clang  8.0.0
CGStmtOpenMP.cpp
Go to the documentation of this file.
1 //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This contains code to emit OpenMP nodes as LLVM code.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGCleanup.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "CodeGenModule.h"
18 #include "TargetInfo.h"
19 #include "clang/AST/Stmt.h"
20 #include "clang/AST/StmtOpenMP.h"
21 #include "clang/AST/DeclOpenMP.h"
22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
25 
26 namespace {
27 /// Lexical scope for OpenMP executable constructs, that handles correct codegen
28 /// for captured expressions.
29 class OMPLexicalScope : public CodeGenFunction::LexicalScope {
30  void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
31  for (const auto *C : S.clauses()) {
32  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
33  if (const auto *PreInit =
34  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
35  for (const auto *I : PreInit->decls()) {
36  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
37  CGF.EmitVarDecl(cast<VarDecl>(*I));
38  } else {
40  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
41  CGF.EmitAutoVarCleanups(Emission);
42  }
43  }
44  }
45  }
46  }
47  }
48  CodeGenFunction::OMPPrivateScope InlinedShareds;
49 
50  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
51  return CGF.LambdaCaptureFields.lookup(VD) ||
52  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
53  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
54  }
55 
56 public:
57  OMPLexicalScope(
59  const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
60  const bool EmitPreInitStmt = true)
61  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
62  InlinedShareds(CGF) {
63  if (EmitPreInitStmt)
64  emitPreInitStmt(CGF, S);
65  if (!CapturedRegion.hasValue())
66  return;
67  assert(S.hasAssociatedStmt() &&
68  "Expected associated statement for inlined directive.");
69  const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
70  for (const auto &C : CS->captures()) {
71  if (C.capturesVariable() || C.capturesVariableByCopy()) {
72  auto *VD = C.getCapturedVar();
73  assert(VD == VD->getCanonicalDecl() &&
74  "Canonical decl must be captured.");
75  DeclRefExpr DRE(
76  CGF.getContext(), const_cast<VarDecl *>(VD),
77  isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
78  InlinedShareds.isGlobalVarCaptured(VD)),
79  VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
80  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
81  return CGF.EmitLValue(&DRE).getAddress();
82  });
83  }
84  }
85  (void)InlinedShareds.Privatize();
86  }
87 };
88 
89 /// Lexical scope for OpenMP parallel construct, that handles correct codegen
90 /// for captured expressions.
91 class OMPParallelScope final : public OMPLexicalScope {
92  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
94  return !(isOpenMPTargetExecutionDirective(Kind) ||
97  }
98 
99 public:
100  OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
101  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
102  EmitPreInitStmt(S)) {}
103 };
104 
105 /// Lexical scope for OpenMP teams construct, that handles correct codegen
106 /// for captured expressions.
107 class OMPTeamsScope final : public OMPLexicalScope {
108  bool EmitPreInitStmt(const OMPExecutableDirective &S) {
110  return !isOpenMPTargetExecutionDirective(Kind) &&
112  }
113 
114 public:
115  OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
116  : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
117  EmitPreInitStmt(S)) {}
118 };
119 
120 /// Private scope for OpenMP loop-based directives, that supports capturing
121 /// of used expression from loop statement.
122 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
123  void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
124  CodeGenFunction::OMPMapVars PreCondVars;
125  for (const auto *E : S.counters()) {
126  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
127  (void)PreCondVars.setVarAddr(
128  CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
129  }
130  (void)PreCondVars.apply(CGF);
131  if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
132  for (const auto *I : PreInits->decls())
133  CGF.EmitVarDecl(cast<VarDecl>(*I));
134  }
135  PreCondVars.restore(CGF);
136  }
137 
138 public:
139  OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
140  : CodeGenFunction::RunCleanupsScope(CGF) {
141  emitPreInitStmt(CGF, S);
142  }
143 };
144 
145 class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
146  CodeGenFunction::OMPPrivateScope InlinedShareds;
147 
148  static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
149  return CGF.LambdaCaptureFields.lookup(VD) ||
150  (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
151  (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
152  cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
153  }
154 
155 public:
156  OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
157  : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
158  InlinedShareds(CGF) {
159  for (const auto *C : S.clauses()) {
160  if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
161  if (const auto *PreInit =
162  cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
163  for (const auto *I : PreInit->decls()) {
164  if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
165  CGF.EmitVarDecl(cast<VarDecl>(*I));
166  } else {
168  CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
169  CGF.EmitAutoVarCleanups(Emission);
170  }
171  }
172  }
173  } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
174  for (const Expr *E : UDP->varlists()) {
175  const Decl *D = cast<DeclRefExpr>(E)->getDecl();
176  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
177  CGF.EmitVarDecl(*OED);
178  }
179  }
180  }
182  CGF.EmitOMPPrivateClause(S, InlinedShareds);
183  if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
184  if (const Expr *E = TG->getReductionRef())
185  CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
186  }
187  const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
188  while (CS) {
189  for (auto &C : CS->captures()) {
190  if (C.capturesVariable() || C.capturesVariableByCopy()) {
191  auto *VD = C.getCapturedVar();
192  assert(VD == VD->getCanonicalDecl() &&
193  "Canonical decl must be captured.");
194  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
195  isCapturedVar(CGF, VD) ||
196  (CGF.CapturedStmtInfo &&
197  InlinedShareds.isGlobalVarCaptured(VD)),
199  C.getLocation());
200  InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
201  return CGF.EmitLValue(&DRE).getAddress();
202  });
203  }
204  }
205  CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
206  }
207  (void)InlinedShareds.Privatize();
208  }
209 };
210 
211 } // namespace
212 
214  const OMPExecutableDirective &S,
215  const RegionCodeGenTy &CodeGen);
216 
218  if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
219  if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
220  OrigVD = OrigVD->getCanonicalDecl();
221  bool IsCaptured =
222  LambdaCaptureFields.lookup(OrigVD) ||
223  (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
224  (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
225  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD), IsCaptured,
226  OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
227  return EmitLValue(&DRE);
228  }
229  }
230  return EmitLValue(E);
231 }
232 
234  ASTContext &C = getContext();
235  llvm::Value *Size = nullptr;
236  auto SizeInChars = C.getTypeSizeInChars(Ty);
237  if (SizeInChars.isZero()) {
238  // getTypeSizeInChars() returns 0 for a VLA.
239  while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
240  VlaSizePair VlaSize = getVLASize(VAT);
241  Ty = VlaSize.Type;
242  Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
243  : VlaSize.NumElts;
244  }
245  SizeInChars = C.getTypeSizeInChars(Ty);
246  if (SizeInChars.isZero())
247  return llvm::ConstantInt::get(SizeTy, /*V=*/0);
248  return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
249  }
250  return CGM.getSize(SizeInChars);
251 }
252 
254  const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
255  const RecordDecl *RD = S.getCapturedRecordDecl();
256  auto CurField = RD->field_begin();
257  auto CurCap = S.captures().begin();
259  E = S.capture_init_end();
260  I != E; ++I, ++CurField, ++CurCap) {
261  if (CurField->hasCapturedVLAType()) {
262  const VariableArrayType *VAT = CurField->getCapturedVLAType();
263  llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
264  CapturedVars.push_back(Val);
265  } else if (CurCap->capturesThis()) {
266  CapturedVars.push_back(CXXThisValue);
267  } else if (CurCap->capturesVariableByCopy()) {
268  llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
269 
270  // If the field is not a pointer, we need to save the actual value
271  // and load it as a void pointer.
272  if (!CurField->getType()->isAnyPointerType()) {
273  ASTContext &Ctx = getContext();
274  Address DstAddr = CreateMemTemp(
275  Ctx.getUIntPtrType(),
276  Twine(CurCap->getCapturedVar()->getName(), ".casted"));
277  LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
278 
279  llvm::Value *SrcAddrVal = EmitScalarConversion(
280  DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
281  Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
282  LValue SrcLV =
283  MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
284 
285  // Store the value using the source type pointer.
286  EmitStoreThroughLValue(RValue::get(CV), SrcLV);
287 
288  // Load the value using the destination type pointer.
289  CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
290  }
291  CapturedVars.push_back(CV);
292  } else {
293  assert(CurCap->capturesVariable() && "Expected capture by reference.");
294  CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
295  }
296  }
297 }
298 
300  QualType DstType, StringRef Name,
301  LValue AddrLV,
302  bool isReferenceType = false) {
303  ASTContext &Ctx = CGF.getContext();
304 
305  llvm::Value *CastedPtr = CGF.EmitScalarConversion(
306  AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
307  Ctx.getPointerType(DstType), Loc);
308  Address TmpAddr =
309  CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
310  .getAddress();
311 
312  // If we are dealing with references we need to return the address of the
313  // reference instead of the reference of the value.
314  if (isReferenceType) {
315  QualType RefType = Ctx.getLValueReferenceType(DstType);
316  llvm::Value *RefVal = TmpAddr.getPointer();
317  TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
318  LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
319  CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
320  }
321 
322  return TmpAddr;
323 }
324 
326  if (T->isLValueReferenceType())
327  return C.getLValueReferenceType(
329  /*SpelledAsLValue=*/false);
330  if (T->isPointerType())
332  if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
333  if (const auto *VLA = dyn_cast<VariableArrayType>(A))
334  return getCanonicalParamType(C, VLA->getElementType());
335  if (!A->isVariablyModifiedType())
336  return C.getCanonicalType(T);
337  }
338  return C.getCanonicalParamType(T);
339 }
340 
341 namespace {
342  /// Contains required data for proper outlined function codegen.
343  struct FunctionOptions {
344  /// Captured statement for which the function is generated.
345  const CapturedStmt *S = nullptr;
346  /// true if cast to/from UIntPtr is required for variables captured by
347  /// value.
348  const bool UIntPtrCastRequired = true;
349  /// true if only casted arguments must be registered as local args or VLA
350  /// sizes.
351  const bool RegisterCastedArgsOnly = false;
352  /// Name of the generated function.
353  const StringRef FunctionName;
354  explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
355  bool RegisterCastedArgsOnly,
356  StringRef FunctionName)
357  : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
358  RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
359  FunctionName(FunctionName) {}
360  };
361 }
362 
363 static llvm::Function *emitOutlinedFunctionPrologue(
364  CodeGenFunction &CGF, FunctionArgList &Args,
365  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
366  &LocalAddrs,
367  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
368  &VLASizes,
369  llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
370  const CapturedDecl *CD = FO.S->getCapturedDecl();
371  const RecordDecl *RD = FO.S->getCapturedRecordDecl();
372  assert(CD->hasBody() && "missing CapturedDecl body");
373 
374  CXXThisValue = nullptr;
375  // Build the argument list.
376  CodeGenModule &CGM = CGF.CGM;
377  ASTContext &Ctx = CGM.getContext();
378  FunctionArgList TargetArgs;
379  Args.append(CD->param_begin(),
380  std::next(CD->param_begin(), CD->getContextParamPosition()));
381  TargetArgs.append(
382  CD->param_begin(),
383  std::next(CD->param_begin(), CD->getContextParamPosition()));
384  auto I = FO.S->captures().begin();
385  FunctionDecl *DebugFunctionDecl = nullptr;
386  if (!FO.UIntPtrCastRequired) {
388  QualType FunctionTy = Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI);
389  DebugFunctionDecl = FunctionDecl::Create(
390  Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
391  SourceLocation(), DeclarationName(), FunctionTy,
392  Ctx.getTrivialTypeSourceInfo(FunctionTy), SC_Static,
393  /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
394  }
395  for (const FieldDecl *FD : RD->fields()) {
396  QualType ArgType = FD->getType();
397  IdentifierInfo *II = nullptr;
398  VarDecl *CapVar = nullptr;
399 
400  // If this is a capture by copy and the type is not a pointer, the outlined
401  // function argument type should be uintptr and the value properly casted to
402  // uintptr. This is necessary given that the runtime library is only able to
403  // deal with pointers. We can pass in the same way the VLA type sizes to the
404  // outlined function.
405  if (FO.UIntPtrCastRequired &&
406  ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
407  I->capturesVariableArrayType()))
408  ArgType = Ctx.getUIntPtrType();
409 
410  if (I->capturesVariable() || I->capturesVariableByCopy()) {
411  CapVar = I->getCapturedVar();
412  II = CapVar->getIdentifier();
413  } else if (I->capturesThis()) {
414  II = &Ctx.Idents.get("this");
415  } else {
416  assert(I->capturesVariableArrayType());
417  II = &Ctx.Idents.get("vla");
418  }
419  if (ArgType->isVariablyModifiedType())
420  ArgType = getCanonicalParamType(Ctx, ArgType);
421  VarDecl *Arg;
422  if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
423  Arg = ParmVarDecl::Create(
424  Ctx, DebugFunctionDecl,
425  CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
426  CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
427  /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
428  } else {
429  Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
430  II, ArgType, ImplicitParamDecl::Other);
431  }
432  Args.emplace_back(Arg);
433  // Do not cast arguments if we emit function with non-original types.
434  TargetArgs.emplace_back(
435  FO.UIntPtrCastRequired
436  ? Arg
437  : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
438  ++I;
439  }
440  Args.append(
441  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
442  CD->param_end());
443  TargetArgs.append(
444  std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
445  CD->param_end());
446 
447  // Create the function declaration.
448  const CGFunctionInfo &FuncInfo =
449  CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
450  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
451 
452  auto *F =
454  FO.FunctionName, &CGM.getModule());
455  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
456  if (CD->isNothrow())
457  F->setDoesNotThrow();
458  F->setDoesNotRecurse();
459 
460  // Generate the function.
461  CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
462  FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
463  unsigned Cnt = CD->getContextParamPosition();
464  I = FO.S->captures().begin();
465  for (const FieldDecl *FD : RD->fields()) {
466  // Do not map arguments if we emit function with non-original types.
467  Address LocalAddr(Address::invalid());
468  if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
469  LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
470  TargetArgs[Cnt]);
471  } else {
472  LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
473  }
474  // If we are capturing a pointer by copy we don't need to do anything, just
475  // use the value that we get from the arguments.
476  if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
477  const VarDecl *CurVD = I->getCapturedVar();
478  // If the variable is a reference we need to materialize it here.
479  if (CurVD->getType()->isReferenceType()) {
480  Address RefAddr = CGF.CreateMemTemp(
481  CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
482  CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
483  /*Volatile=*/false, CurVD->getType());
484  LocalAddr = RefAddr;
485  }
486  if (!FO.RegisterCastedArgsOnly)
487  LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
488  ++Cnt;
489  ++I;
490  continue;
491  }
492 
493  LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
495  if (FD->hasCapturedVLAType()) {
496  if (FO.UIntPtrCastRequired) {
497  ArgLVal = CGF.MakeAddrLValue(
498  castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
499  Args[Cnt]->getName(), ArgLVal),
501  }
502  llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
503  const VariableArrayType *VAT = FD->getCapturedVLAType();
504  VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
505  } else if (I->capturesVariable()) {
506  const VarDecl *Var = I->getCapturedVar();
507  QualType VarTy = Var->getType();
508  Address ArgAddr = ArgLVal.getAddress();
509  if (!VarTy->isReferenceType()) {
510  if (ArgLVal.getType()->isLValueReferenceType()) {
511  ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
512  } else if (!VarTy->isVariablyModifiedType() ||
513  !VarTy->isPointerType()) {
514  assert(ArgLVal.getType()->isPointerType());
515  ArgAddr = CGF.EmitLoadOfPointer(
516  ArgAddr, ArgLVal.getType()->castAs<PointerType>());
517  }
518  }
519  if (!FO.RegisterCastedArgsOnly) {
520  LocalAddrs.insert(
521  {Args[Cnt],
522  {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
523  }
524  } else if (I->capturesVariableByCopy()) {
525  assert(!FD->getType()->isAnyPointerType() &&
526  "Not expecting a captured pointer.");
527  const VarDecl *Var = I->getCapturedVar();
528  QualType VarTy = Var->getType();
529  LocalAddrs.insert(
530  {Args[Cnt],
531  {Var, FO.UIntPtrCastRequired
532  ? castValueFromUintptr(CGF, I->getLocation(),
533  FD->getType(), Args[Cnt]->getName(),
534  ArgLVal, VarTy->isReferenceType())
535  : ArgLVal.getAddress()}});
536  } else {
537  // If 'this' is captured, load it into CXXThisValue.
538  assert(I->capturesThis());
539  CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
540  LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
541  }
542  ++Cnt;
543  ++I;
544  }
545 
546  return F;
547 }
548 
549 llvm::Function *
551  assert(
552  CapturedStmtInfo &&
553  "CapturedStmtInfo should be set when generating the captured function");
554  const CapturedDecl *CD = S.getCapturedDecl();
555  // Build the argument list.
556  bool NeedWrapperFunction =
557  getDebugInfo() &&
558  CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
559  FunctionArgList Args;
560  llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
561  llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
562  SmallString<256> Buffer;
563  llvm::raw_svector_ostream Out(Buffer);
564  Out << CapturedStmtInfo->getHelperName();
565  if (NeedWrapperFunction)
566  Out << "_debug__";
567  FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
568  Out.str());
569  llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
570  VLASizes, CXXThisValue, FO);
571  for (const auto &LocalAddrPair : LocalAddrs) {
572  if (LocalAddrPair.second.first) {
573  setAddrOfLocalVar(LocalAddrPair.second.first,
574  LocalAddrPair.second.second);
575  }
576  }
577  for (const auto &VLASizePair : VLASizes)
578  VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
579  PGO.assignRegionCounters(GlobalDecl(CD), F);
580  CapturedStmtInfo->EmitBody(*this, CD->getBody());
581  FinishFunction(CD->getBodyRBrace());
582  if (!NeedWrapperFunction)
583  return F;
584 
585  FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
586  /*RegisterCastedArgsOnly=*/true,
587  CapturedStmtInfo->getHelperName());
588  CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
589  WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
590  Args.clear();
591  LocalAddrs.clear();
592  VLASizes.clear();
593  llvm::Function *WrapperF =
594  emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
595  WrapperCGF.CXXThisValue, WrapperFO);
597  for (const auto *Arg : Args) {
599  auto I = LocalAddrs.find(Arg);
600  if (I != LocalAddrs.end()) {
601  LValue LV = WrapperCGF.MakeAddrLValue(
602  I->second.second,
603  I->second.first ? I->second.first->getType() : Arg->getType(),
605  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
606  } else {
607  auto EI = VLASizes.find(Arg);
608  if (EI != VLASizes.end()) {
609  CallArg = EI->second.second;
610  } else {
611  LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
612  Arg->getType(),
614  CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
615  }
616  }
617  CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
618  }
619  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
620  F, CallArgs);
621  WrapperCGF.FinishFunction();
622  return WrapperF;
623 }
624 
625 //===----------------------------------------------------------------------===//
626 // OpenMP Directive Emission
627 //===----------------------------------------------------------------------===//
629  Address DestAddr, Address SrcAddr, QualType OriginalType,
630  const llvm::function_ref<void(Address, Address)> CopyGen) {
631  // Perform element-by-element initialization.
632  QualType ElementTy;
633 
634  // Drill down to the base element type on both arrays.
635  const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
636  llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
637  SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
638 
639  llvm::Value *SrcBegin = SrcAddr.getPointer();
640  llvm::Value *DestBegin = DestAddr.getPointer();
641  // Cast from pointer to array type to pointer to single element.
642  llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
643  // The basic structure here is a while-do loop.
644  llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
645  llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
646  llvm::Value *IsEmpty =
647  Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
648  Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
649 
650  // Enter the loop body, making that address the current address.
651  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
652  EmitBlock(BodyBB);
653 
654  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
655 
656  llvm::PHINode *SrcElementPHI =
657  Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
658  SrcElementPHI->addIncoming(SrcBegin, EntryBB);
659  Address SrcElementCurrent =
660  Address(SrcElementPHI,
661  SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
662 
663  llvm::PHINode *DestElementPHI =
664  Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
665  DestElementPHI->addIncoming(DestBegin, EntryBB);
666  Address DestElementCurrent =
667  Address(DestElementPHI,
668  DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
669 
670  // Emit copy.
671  CopyGen(DestElementCurrent, SrcElementCurrent);
672 
673  // Shift the address forward by one element.
674  llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
675  DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
676  llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
677  SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
678  // Check whether we've reached the end.
679  llvm::Value *Done =
680  Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
681  Builder.CreateCondBr(Done, DoneBB, BodyBB);
682  DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
683  SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
684 
685  // Done.
686  EmitBlock(DoneBB, /*IsFinished=*/true);
687 }
688 
689 void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
690  Address SrcAddr, const VarDecl *DestVD,
691  const VarDecl *SrcVD, const Expr *Copy) {
692  if (OriginalType->isArrayType()) {
693  const auto *BO = dyn_cast<BinaryOperator>(Copy);
694  if (BO && BO->getOpcode() == BO_Assign) {
695  // Perform simple memcpy for simple copying.
696  LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
697  LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
698  EmitAggregateAssign(Dest, Src, OriginalType);
699  } else {
700  // For arrays with complex element types perform element by element
701  // copying.
702  EmitOMPAggregateAssign(
703  DestAddr, SrcAddr, OriginalType,
704  [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
705  // Working with the single array element, so have to remap
706  // destination and source variables to corresponding array
707  // elements.
709  Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
710  Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
711  (void)Remap.Privatize();
712  EmitIgnoredExpr(Copy);
713  });
714  }
715  } else {
716  // Remap pseudo source variable to private copy.
718  Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
719  Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
720  (void)Remap.Privatize();
721  // Emit copying of the whole variable.
722  EmitIgnoredExpr(Copy);
723  }
724 }
725 
727  OMPPrivateScope &PrivateScope) {
728  if (!HaveInsertPoint())
729  return false;
730  bool FirstprivateIsLastprivate = false;
731  llvm::DenseSet<const VarDecl *> Lastprivates;
732  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
733  for (const auto *D : C->varlists())
734  Lastprivates.insert(
735  cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
736  }
737  llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
739  getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
740  // Force emission of the firstprivate copy if the directive does not emit
741  // outlined function, like omp for, omp simd, omp distribute etc.
742  bool MustEmitFirstprivateCopy =
743  CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
744  for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
745  auto IRef = C->varlist_begin();
746  auto InitsRef = C->inits().begin();
747  for (const Expr *IInit : C->private_copies()) {
748  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
749  bool ThisFirstprivateIsLastprivate =
750  Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
751  const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
752  if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
753  !FD->getType()->isReferenceType()) {
754  EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
755  ++IRef;
756  ++InitsRef;
757  continue;
758  }
759  FirstprivateIsLastprivate =
760  FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
761  if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
762  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
763  const auto *VDInit =
764  cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
765  bool IsRegistered;
766  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
767  /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
768  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
769  LValue OriginalLVal = EmitLValue(&DRE);
770  QualType Type = VD->getType();
771  if (Type->isArrayType()) {
772  // Emit VarDecl with copy init for arrays.
773  // Get the address of the original variable captured in current
774  // captured region.
775  IsRegistered = PrivateScope.addPrivate(
776  OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
777  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
778  const Expr *Init = VD->getInit();
779  if (!isa<CXXConstructExpr>(Init) ||
780  isTrivialInitializer(Init)) {
781  // Perform simple memcpy.
782  LValue Dest =
783  MakeAddrLValue(Emission.getAllocatedAddress(), Type);
784  EmitAggregateAssign(Dest, OriginalLVal, Type);
785  } else {
786  EmitOMPAggregateAssign(
787  Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
788  Type,
789  [this, VDInit, Init](Address DestElement,
790  Address SrcElement) {
791  // Clean up any temporaries needed by the
792  // initialization.
793  RunCleanupsScope InitScope(*this);
794  // Emit initialization for single element.
795  setAddrOfLocalVar(VDInit, SrcElement);
796  EmitAnyExprToMem(Init, DestElement,
797  Init->getType().getQualifiers(),
798  /*IsInitializer*/ false);
799  LocalDeclMap.erase(VDInit);
800  });
801  }
802  EmitAutoVarCleanups(Emission);
803  return Emission.getAllocatedAddress();
804  });
805  } else {
806  Address OriginalAddr = OriginalLVal.getAddress();
807  IsRegistered = PrivateScope.addPrivate(
808  OrigVD, [this, VDInit, OriginalAddr, VD]() {
809  // Emit private VarDecl with copy init.
810  // Remap temp VDInit variable to the address of the original
811  // variable (for proper handling of captured global variables).
812  setAddrOfLocalVar(VDInit, OriginalAddr);
813  EmitDecl(*VD);
814  LocalDeclMap.erase(VDInit);
815  return GetAddrOfLocalVar(VD);
816  });
817  }
818  assert(IsRegistered &&
819  "firstprivate var already registered as private");
820  // Silence the warning about unused variable.
821  (void)IsRegistered;
822  }
823  ++IRef;
824  ++InitsRef;
825  }
826  }
827  return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
828 }
829 
831  const OMPExecutableDirective &D,
832  CodeGenFunction::OMPPrivateScope &PrivateScope) {
833  if (!HaveInsertPoint())
834  return;
835  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
836  for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
837  auto IRef = C->varlist_begin();
838  for (const Expr *IInit : C->private_copies()) {
839  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
840  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
841  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
842  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
843  // Emit private VarDecl with copy init.
844  EmitDecl(*VD);
845  return GetAddrOfLocalVar(VD);
846  });
847  assert(IsRegistered && "private var already registered as private");
848  // Silence the warning about unused variable.
849  (void)IsRegistered;
850  }
851  ++IRef;
852  }
853  }
854 }
855 
857  if (!HaveInsertPoint())
858  return false;
859  // threadprivate_var1 = master_threadprivate_var1;
860  // operator=(threadprivate_var2, master_threadprivate_var2);
861  // ...
862  // __kmpc_barrier(&loc, global_tid);
864  llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
865  for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
866  auto IRef = C->varlist_begin();
867  auto ISrcRef = C->source_exprs().begin();
868  auto IDestRef = C->destination_exprs().begin();
869  for (const Expr *AssignOp : C->assignment_ops()) {
870  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
871  QualType Type = VD->getType();
872  if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
873  // Get the address of the master variable. If we are emitting code with
874  // TLS support, the address is passed from the master as field in the
875  // captured declaration.
876  Address MasterAddr = Address::invalid();
877  if (getLangOpts().OpenMPUseTLS &&
878  getContext().getTargetInfo().isTLSSupported()) {
879  assert(CapturedStmtInfo->lookup(VD) &&
880  "Copyin threadprivates should have been captured!");
881  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD), true,
882  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
883  MasterAddr = EmitLValue(&DRE).getAddress();
884  LocalDeclMap.erase(VD);
885  } else {
886  MasterAddr =
887  Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
888  : CGM.GetAddrOfGlobal(VD),
889  getContext().getDeclAlign(VD));
890  }
891  // Get the address of the threadprivate variable.
892  Address PrivateAddr = EmitLValue(*IRef).getAddress();
893  if (CopiedVars.size() == 1) {
894  // At first check if current thread is a master thread. If it is, no
895  // need to copy data.
896  CopyBegin = createBasicBlock("copyin.not.master");
897  CopyEnd = createBasicBlock("copyin.not.master.end");
898  Builder.CreateCondBr(
899  Builder.CreateICmpNE(
900  Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
901  Builder.CreatePtrToInt(PrivateAddr.getPointer(),
902  CGM.IntPtrTy)),
903  CopyBegin, CopyEnd);
904  EmitBlock(CopyBegin);
905  }
906  const auto *SrcVD =
907  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
908  const auto *DestVD =
909  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
910  EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
911  }
912  ++IRef;
913  ++ISrcRef;
914  ++IDestRef;
915  }
916  }
917  if (CopyEnd) {
918  // Exit out of copying procedure for non-master thread.
919  EmitBlock(CopyEnd, /*IsFinished=*/true);
920  return true;
921  }
922  return false;
923 }
924 
926  const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
927  if (!HaveInsertPoint())
928  return false;
929  bool HasAtLeastOneLastprivate = false;
932  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
933  for (const Expr *C : LoopDirective->counters()) {
934  SIMDLCVs.insert(
935  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
936  }
937  }
938  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
939  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
940  HasAtLeastOneLastprivate = true;
942  !getLangOpts().OpenMPSimd)
943  break;
944  auto IRef = C->varlist_begin();
945  auto IDestRef = C->destination_exprs().begin();
946  for (const Expr *IInit : C->private_copies()) {
947  // Keep the address of the original variable for future update at the end
948  // of the loop.
949  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
950  // Taskloops do not require additional initialization, it is done in
951  // runtime support library.
952  if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
953  const auto *DestVD =
954  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
955  PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
956  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
957  /*RefersToEnclosingVariableOrCapture=*/
958  CapturedStmtInfo->lookup(OrigVD) != nullptr,
959  (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
960  return EmitLValue(&DRE).getAddress();
961  });
962  // Check if the variable is also a firstprivate: in this case IInit is
963  // not generated. Initialization of this variable will happen in codegen
964  // for 'firstprivate' clause.
965  if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
966  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
967  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
968  // Emit private VarDecl with copy init.
969  EmitDecl(*VD);
970  return GetAddrOfLocalVar(VD);
971  });
972  assert(IsRegistered &&
973  "lastprivate var already registered as private");
974  (void)IsRegistered;
975  }
976  }
977  ++IRef;
978  ++IDestRef;
979  }
980  }
981  return HasAtLeastOneLastprivate;
982 }
983 
985  const OMPExecutableDirective &D, bool NoFinals,
986  llvm::Value *IsLastIterCond) {
987  if (!HaveInsertPoint())
988  return;
989  // Emit following code:
990  // if (<IsLastIterCond>) {
991  // orig_var1 = private_orig_var1;
992  // ...
993  // orig_varn = private_orig_varn;
994  // }
995  llvm::BasicBlock *ThenBB = nullptr;
996  llvm::BasicBlock *DoneBB = nullptr;
997  if (IsLastIterCond) {
998  ThenBB = createBasicBlock(".omp.lastprivate.then");
999  DoneBB = createBasicBlock(".omp.lastprivate.done");
1000  Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
1001  EmitBlock(ThenBB);
1002  }
1003  llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
1004  llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
1005  if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
1006  auto IC = LoopDirective->counters().begin();
1007  for (const Expr *F : LoopDirective->finals()) {
1008  const auto *D =
1009  cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
1010  if (NoFinals)
1011  AlreadyEmittedVars.insert(D);
1012  else
1013  LoopCountersAndUpdates[D] = F;
1014  ++IC;
1015  }
1016  }
1017  for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
1018  auto IRef = C->varlist_begin();
1019  auto ISrcRef = C->source_exprs().begin();
1020  auto IDestRef = C->destination_exprs().begin();
1021  for (const Expr *AssignOp : C->assignment_ops()) {
1022  const auto *PrivateVD =
1023  cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
1024  QualType Type = PrivateVD->getType();
1025  const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
1026  if (AlreadyEmittedVars.insert(CanonicalVD).second) {
1027  // If lastprivate variable is a loop control variable for loop-based
1028  // directive, update its value before copyin back to original
1029  // variable.
1030  if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
1031  EmitIgnoredExpr(FinalExpr);
1032  const auto *SrcVD =
1033  cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
1034  const auto *DestVD =
1035  cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
1036  // Get the address of the original variable.
1037  Address OriginalAddr = GetAddrOfLocalVar(DestVD);
1038  // Get the address of the private variable.
1039  Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
1040  if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
1041  PrivateAddr =
1042  Address(Builder.CreateLoad(PrivateAddr),
1043  getNaturalTypeAlignment(RefTy->getPointeeType()));
1044  EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
1045  }
1046  ++IRef;
1047  ++ISrcRef;
1048  ++IDestRef;
1049  }
1050  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1051  EmitIgnoredExpr(PostUpdate);
1052  }
1053  if (IsLastIterCond)
1054  EmitBlock(DoneBB, /*IsFinished=*/true);
1055 }
1056 
1058  const OMPExecutableDirective &D,
1059  CodeGenFunction::OMPPrivateScope &PrivateScope) {
1060  if (!HaveInsertPoint())
1061  return;
1064  SmallVector<const Expr *, 4> ReductionOps;
1067  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1068  auto IPriv = C->privates().begin();
1069  auto IRed = C->reduction_ops().begin();
1070  auto ILHS = C->lhs_exprs().begin();
1071  auto IRHS = C->rhs_exprs().begin();
1072  for (const Expr *Ref : C->varlists()) {
1073  Shareds.emplace_back(Ref);
1074  Privates.emplace_back(*IPriv);
1075  ReductionOps.emplace_back(*IRed);
1076  LHSs.emplace_back(*ILHS);
1077  RHSs.emplace_back(*IRHS);
1078  std::advance(IPriv, 1);
1079  std::advance(IRed, 1);
1080  std::advance(ILHS, 1);
1081  std::advance(IRHS, 1);
1082  }
1083  }
1084  ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
1085  unsigned Count = 0;
1086  auto ILHS = LHSs.begin();
1087  auto IRHS = RHSs.begin();
1088  auto IPriv = Privates.begin();
1089  for (const Expr *IRef : Shareds) {
1090  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
1091  // Emit private VarDecl with reduction init.
1092  RedCG.emitSharedLValue(*this, Count);
1093  RedCG.emitAggregateType(*this, Count);
1094  AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
1095  RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
1096  RedCG.getSharedLValue(Count),
1097  [&Emission](CodeGenFunction &CGF) {
1098  CGF.EmitAutoVarInit(Emission);
1099  return true;
1100  });
1101  EmitAutoVarCleanups(Emission);
1102  Address BaseAddr = RedCG.adjustPrivateAddress(
1103  *this, Count, Emission.getAllocatedAddress());
1104  bool IsRegistered = PrivateScope.addPrivate(
1105  RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
1106  assert(IsRegistered && "private var already registered as private");
1107  // Silence the warning about unused variable.
1108  (void)IsRegistered;
1109 
1110  const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
1111  const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
1112  QualType Type = PrivateVD->getType();
1113  bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
1114  if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
1115  // Store the address of the original variable associated with the LHS
1116  // implicit variable.
1117  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1118  return RedCG.getSharedLValue(Count).getAddress();
1119  });
1120  PrivateScope.addPrivate(
1121  RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
1122  } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
1123  isa<ArraySubscriptExpr>(IRef)) {
1124  // Store the address of the original variable associated with the LHS
1125  // implicit variable.
1126  PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
1127  return RedCG.getSharedLValue(Count).getAddress();
1128  });
1129  PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
1130  return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
1131  ConvertTypeForMem(RHSVD->getType()),
1132  "rhs.begin");
1133  });
1134  } else {
1135  QualType Type = PrivateVD->getType();
1136  bool IsArray = getContext().getAsArrayType(Type) != nullptr;
1137  Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
1138  // Store the address of the original variable associated with the LHS
1139  // implicit variable.
1140  if (IsArray) {
1141  OriginalAddr = Builder.CreateElementBitCast(
1142  OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
1143  }
1144  PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
1145  PrivateScope.addPrivate(
1146  RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
1147  return IsArray
1148  ? Builder.CreateElementBitCast(
1149  GetAddrOfLocalVar(PrivateVD),
1150  ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
1151  : GetAddrOfLocalVar(PrivateVD);
1152  });
1153  }
1154  ++ILHS;
1155  ++IRHS;
1156  ++IPriv;
1157  ++Count;
1158  }
1159 }
1160 
1162  const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
1163  if (!HaveInsertPoint())
1164  return;
1169  bool HasAtLeastOneReduction = false;
1170  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1171  HasAtLeastOneReduction = true;
1172  Privates.append(C->privates().begin(), C->privates().end());
1173  LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1174  RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1175  ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1176  }
1177  if (HasAtLeastOneReduction) {
1178  bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
1180  ReductionKind == OMPD_simd;
1181  bool SimpleReduction = ReductionKind == OMPD_simd;
1182  // Emit nowait reduction if nowait clause is present or directive is a
1183  // parallel directive (it always has implicit barrier).
1184  CGM.getOpenMPRuntime().emitReduction(
1185  *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
1186  {WithNowait, SimpleReduction, ReductionKind});
1187  }
1188 }
1189 
1191  CodeGenFunction &CGF, const OMPExecutableDirective &D,
1192  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1193  if (!CGF.HaveInsertPoint())
1194  return;
1195  llvm::BasicBlock *DoneBB = nullptr;
1196  for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1197  if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
1198  if (!DoneBB) {
1199  if (llvm::Value *Cond = CondGen(CGF)) {
1200  // If the first post-update expression is found, emit conditional
1201  // block if it was requested.
1202  llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
1203  DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
1204  CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1205  CGF.EmitBlock(ThenBB);
1206  }
1207  }
1208  CGF.EmitIgnoredExpr(PostUpdate);
1209  }
1210  }
1211  if (DoneBB)
1212  CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
1213 }
1214 
1215 namespace {
1216 /// Codegen lambda for appending distribute lower and upper bounds to outlined
1217 /// parallel function. This is necessary for combined constructs such as
1218 /// 'distribute parallel for'
1219 typedef llvm::function_ref<void(CodeGenFunction &,
1220  const OMPExecutableDirective &,
1222  CodeGenBoundParametersTy;
1223 } // anonymous namespace
1224 
1226  CodeGenFunction &CGF, const OMPExecutableDirective &S,
1227  OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1228  const CodeGenBoundParametersTy &CodeGenBoundParameters) {
1229  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
1230  llvm::Value *OutlinedFn =
1231  CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
1232  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1233  if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
1234  CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
1235  llvm::Value *NumThreads =
1236  CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
1237  /*IgnoreResultAssign=*/true);
1238  CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
1239  CGF, NumThreads, NumThreadsClause->getBeginLoc());
1240  }
1241  if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
1242  CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
1243  CGF.CGM.getOpenMPRuntime().emitProcBindClause(
1244  CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
1245  }
1246  const Expr *IfCond = nullptr;
1247  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
1248  if (C->getNameModifier() == OMPD_unknown ||
1249  C->getNameModifier() == OMPD_parallel) {
1250  IfCond = C->getCondition();
1251  break;
1252  }
1253  }
1254 
1255  OMPParallelScope Scope(CGF, S);
1257  // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
1258  // lower and upper bounds with the pragma 'for' chunking mechanism.
1259  // The following lambda takes care of appending the lower and upper bound
1260  // parameters when necessary
1261  CodeGenBoundParameters(CGF, S, CapturedVars);
1262  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
1263  CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
1264  CapturedVars, IfCond);
1265 }
1266 
1268  const OMPExecutableDirective &,
1270 
1272  // Emit parallel region as a standalone region.
1273  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1274  Action.Enter(CGF);
1275  OMPPrivateScope PrivateScope(CGF);
1276  bool Copyins = CGF.EmitOMPCopyinClause(S);
1277  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
1278  if (Copyins) {
1279  // Emit implicit barrier to synchronize threads and avoid data races on
1280  // propagation master's thread values of threadprivate variables to local
1281  // instances of that variables of all other implicit threads.
1282  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
1283  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
1284  /*ForceSimpleCall=*/true);
1285  }
1286  CGF.EmitOMPPrivateClause(S, PrivateScope);
1287  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
1288  (void)PrivateScope.Privatize();
1289  CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
1290  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
1291  };
1292  emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
1295  [](CodeGenFunction &) { return nullptr; });
1296 }
1297 
1299  JumpDest LoopExit) {
1300  RunCleanupsScope BodyScope(*this);
1301  // Update counters values on current iteration.
1302  for (const Expr *UE : D.updates())
1303  EmitIgnoredExpr(UE);
1304  // Update the linear variables.
1305  // In distribute directives only loop counters may be marked as linear, no
1306  // need to generate the code for them.
1308  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1309  for (const Expr *UE : C->updates())
1310  EmitIgnoredExpr(UE);
1311  }
1312  }
1313 
1314  // On a continue in the body, jump to the end.
1315  JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
1316  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1317  // Emit loop body.
1318  EmitStmt(D.getBody());
1319  // The end (updates/cleanups).
1320  EmitBlock(Continue.getBlock());
1321  BreakContinueStack.pop_back();
1322 }
1323 
1325  const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
1326  const Expr *IncExpr,
1327  const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
1328  const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
1329  auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
1330 
1331  // Start the loop with a block that tests the condition.
1332  auto CondBlock = createBasicBlock("omp.inner.for.cond");
1333  EmitBlock(CondBlock);
1334  const SourceRange R = S.getSourceRange();
1335  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1336  SourceLocToDebugLoc(R.getEnd()));
1337 
1338  // If there are any cleanups between here and the loop-exit scope,
1339  // create a block to stage a loop exit along.
1340  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1341  if (RequiresCleanup)
1342  ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
1343 
1344  llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
1345 
1346  // Emit condition.
1347  EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
1348  if (ExitBlock != LoopExit.getBlock()) {
1349  EmitBlock(ExitBlock);
1350  EmitBranchThroughCleanup(LoopExit);
1351  }
1352 
1353  EmitBlock(LoopBody);
1354  incrementProfileCounter(&S);
1355 
1356  // Create a block for the increment.
1357  JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
1358  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1359 
1360  BodyGen(*this);
1361 
1362  // Emit "IV = IV + 1" and a back-edge to the condition block.
1363  EmitBlock(Continue.getBlock());
1364  EmitIgnoredExpr(IncExpr);
1365  PostIncGen(*this);
1366  BreakContinueStack.pop_back();
1367  EmitBranch(CondBlock);
1368  LoopStack.pop();
1369  // Emit the fall-through block.
1370  EmitBlock(LoopExit.getBlock());
1371 }
1372 
1374  if (!HaveInsertPoint())
1375  return false;
1376  // Emit inits for the linear variables.
1377  bool HasLinears = false;
1378  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1379  for (const Expr *Init : C->inits()) {
1380  HasLinears = true;
1381  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1382  if (const auto *Ref =
1383  dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
1384  AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
1385  const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1386  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1387  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1388  VD->getInit()->getType(), VK_LValue,
1389  VD->getInit()->getExprLoc());
1390  EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
1391  VD->getType()),
1392  /*capturedByInit=*/false);
1393  EmitAutoVarCleanups(Emission);
1394  } else {
1395  EmitVarDecl(*VD);
1396  }
1397  }
1398  // Emit the linear steps for the linear clauses.
1399  // If a step is not constant, it is pre-calculated before the loop.
1400  if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1401  if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1402  EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
1403  // Emit calculation of the linear step.
1404  EmitIgnoredExpr(CS);
1405  }
1406  }
1407  return HasLinears;
1408 }
1409 
1411  const OMPLoopDirective &D,
1412  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1413  if (!HaveInsertPoint())
1414  return;
1415  llvm::BasicBlock *DoneBB = nullptr;
1416  // Emit the final values of the linear variables.
1417  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1418  auto IC = C->varlist_begin();
1419  for (const Expr *F : C->finals()) {
1420  if (!DoneBB) {
1421  if (llvm::Value *Cond = CondGen(*this)) {
1422  // If the first post-update expression is found, emit conditional
1423  // block if it was requested.
1424  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
1425  DoneBB = createBasicBlock(".omp.linear.pu.done");
1426  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1427  EmitBlock(ThenBB);
1428  }
1429  }
1430  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1431  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(OrigVD),
1432  CapturedStmtInfo->lookup(OrigVD) != nullptr,
1433  (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
1434  Address OrigAddr = EmitLValue(&DRE).getAddress();
1435  CodeGenFunction::OMPPrivateScope VarScope(*this);
1436  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1437  (void)VarScope.Privatize();
1438  EmitIgnoredExpr(F);
1439  ++IC;
1440  }
1441  if (const Expr *PostUpdate = C->getPostUpdateExpr())
1442  EmitIgnoredExpr(PostUpdate);
1443  }
1444  if (DoneBB)
1445  EmitBlock(DoneBB, /*IsFinished=*/true);
1446 }
1447 
1449  const OMPExecutableDirective &D) {
1450  if (!CGF.HaveInsertPoint())
1451  return;
1452  for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
1453  unsigned ClauseAlignment = 0;
1454  if (const Expr *AlignmentExpr = Clause->getAlignment()) {
1455  auto *AlignmentCI =
1456  cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
1457  ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
1458  }
1459  for (const Expr *E : Clause->varlists()) {
1460  unsigned Alignment = ClauseAlignment;
1461  if (Alignment == 0) {
1462  // OpenMP [2.8.1, Description]
1463  // If no optional parameter is specified, implementation-defined default
1464  // alignments for SIMD instructions on the target platforms are assumed.
1465  Alignment =
1466  CGF.getContext()
1468  E->getType()->getPointeeType()))
1469  .getQuantity();
1470  }
1471  assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1472  "alignment is not power of 2");
1473  if (Alignment != 0) {
1474  llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
1476  PtrValue, E, /*No second loc needed*/ SourceLocation(), Alignment);
1477  }
1478  }
1479  }
1480 }
1481 
1483  const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
1484  if (!HaveInsertPoint())
1485  return;
1486  auto I = S.private_counters().begin();
1487  for (const Expr *E : S.counters()) {
1488  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1489  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1490  // Emit var without initialization.
1491  AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
1492  EmitAutoVarCleanups(VarEmission);
1493  LocalDeclMap.erase(PrivateVD);
1494  (void)LoopScope.addPrivate(VD, [&VarEmission]() {
1495  return VarEmission.getAllocatedAddress();
1496  });
1497  if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
1498  VD->hasGlobalStorage()) {
1499  (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
1500  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(VD),
1501  LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
1502  E->getType(), VK_LValue, E->getExprLoc());
1503  return EmitLValue(&DRE).getAddress();
1504  });
1505  } else {
1506  (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
1507  return VarEmission.getAllocatedAddress();
1508  });
1509  }
1510  ++I;
1511  }
1512  // Privatize extra loop counters used in loops for ordered(n) clauses.
1513  for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
1514  if (!C->getNumForLoops())
1515  continue;
1516  for (unsigned I = S.getCollapsedNumber(),
1517  E = C->getLoopNumIterations().size();
1518  I < E; ++I) {
1519  const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
1520  const auto *VD = cast<VarDecl>(DRE->getDecl());
1521  // Override only those variables that are really emitted already.
1522  if (LocalDeclMap.count(VD)) {
1523  (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
1524  return CreateMemTemp(DRE->getType(), VD->getName());
1525  });
1526  }
1527  }
1528  }
1529 }
1530 
1531 static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
1532  const Expr *Cond, llvm::BasicBlock *TrueBlock,
1533  llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1534  if (!CGF.HaveInsertPoint())
1535  return;
1536  {
1537  CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
1538  CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
1539  (void)PreCondScope.Privatize();
1540  // Get initial values of real counters.
1541  for (const Expr *I : S.inits()) {
1542  CGF.EmitIgnoredExpr(I);
1543  }
1544  }
1545  // Check that loop is executed at least one time.
1546  CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
1547 }
1548 
1550  const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
1551  if (!HaveInsertPoint())
1552  return;
1555  const auto *LoopDirective = cast<OMPLoopDirective>(&D);
1556  for (const Expr *C : LoopDirective->counters()) {
1557  SIMDLCVs.insert(
1558  cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
1559  }
1560  }
1561  for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
1562  auto CurPrivate = C->privates().begin();
1563  for (const Expr *E : C->varlists()) {
1564  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
1565  const auto *PrivateVD =
1566  cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1567  if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
1568  bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
1569  // Emit private VarDecl with copy init.
1570  EmitVarDecl(*PrivateVD);
1571  return GetAddrOfLocalVar(PrivateVD);
1572  });
1573  assert(IsRegistered && "linear var already registered as private");
1574  // Silence the warning about unused variable.
1575  (void)IsRegistered;
1576  } else {
1577  EmitVarDecl(*PrivateVD);
1578  }
1579  ++CurPrivate;
1580  }
1581  }
1582 }
1583 
1585  const OMPExecutableDirective &D,
1586  bool IsMonotonic) {
1587  if (!CGF.HaveInsertPoint())
1588  return;
1589  if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
1590  RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
1591  /*ignoreResult=*/true);
1592  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1593  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1594  // In presence of finite 'safelen', it may be unsafe to mark all
1595  // the memory instructions parallel, because loop-carried
1596  // dependences of 'safelen' iterations are possible.
1597  if (!IsMonotonic)
1599  } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
1600  RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
1601  /*ignoreResult=*/true);
1602  auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
1603  CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
1604  // In presence of finite 'safelen', it may be unsafe to mark all
1605  // the memory instructions parallel, because loop-carried
1606  // dependences of 'safelen' iterations are possible.
1607  CGF.LoopStack.setParallel(/*Enable=*/false);
1608  }
1609 }
1610 
1612  bool IsMonotonic) {
1613  // Walk clauses and process safelen/lastprivate.
1614  LoopStack.setParallel(!IsMonotonic);
1615  LoopStack.setVectorizeEnable();
1616  emitSimdlenSafelenClause(*this, D, IsMonotonic);
1617 }
1618 
1620  const OMPLoopDirective &D,
1621  const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
1622  if (!HaveInsertPoint())
1623  return;
1624  llvm::BasicBlock *DoneBB = nullptr;
1625  auto IC = D.counters().begin();
1626  auto IPC = D.private_counters().begin();
1627  for (const Expr *F : D.finals()) {
1628  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1629  const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1630  const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
1631  if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
1632  OrigVD->hasGlobalStorage() || CED) {
1633  if (!DoneBB) {
1634  if (llvm::Value *Cond = CondGen(*this)) {
1635  // If the first post-update expression is found, emit conditional
1636  // block if it was requested.
1637  llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
1638  DoneBB = createBasicBlock(".omp.final.done");
1639  Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1640  EmitBlock(ThenBB);
1641  }
1642  }
1643  Address OrigAddr = Address::invalid();
1644  if (CED) {
1645  OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1646  } else {
1647  DeclRefExpr DRE(getContext(), const_cast<VarDecl *>(PrivateVD),
1648  /*RefersToEnclosingVariableOrCapture=*/false,
1649  (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
1650  OrigAddr = EmitLValue(&DRE).getAddress();
1651  }
1652  OMPPrivateScope VarScope(*this);
1653  VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
1654  (void)VarScope.Privatize();
1655  EmitIgnoredExpr(F);
1656  }
1657  ++IC;
1658  ++IPC;
1659  }
1660  if (DoneBB)
1661  EmitBlock(DoneBB, /*IsFinished=*/true);
1662 }
1663 
1665  const OMPLoopDirective &S,
1667  CGF.EmitOMPLoopBody(S, LoopExit);
1668  CGF.EmitStopPoint(&S);
1669 }
1670 
1671 /// Emit a helper variable and return corresponding lvalue.
1673  const DeclRefExpr *Helper) {
1674  auto VDecl = cast<VarDecl>(Helper->getDecl());
1675  CGF.EmitVarDecl(*VDecl);
1676  return CGF.EmitLValue(Helper);
1677 }
1678 
1680  PrePostActionTy &Action) {
1681  Action.Enter(CGF);
1683  "Expected simd directive");
1684  OMPLoopScope PreInitScope(CGF, S);
1685  // if (PreCond) {
1686  // for (IV in 0..LastIteration) BODY;
1687  // <Final counter/linear vars updates>;
1688  // }
1689  //
1693  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
1694  (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
1695  }
1696 
1697  // Emit: if (PreCond) - begin.
1698  // If the condition constant folds and can be elided, avoid emitting the
1699  // whole loop.
1700  bool CondConstant;
1701  llvm::BasicBlock *ContBlock = nullptr;
1702  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
1703  if (!CondConstant)
1704  return;
1705  } else {
1706  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
1707  ContBlock = CGF.createBasicBlock("simd.if.end");
1708  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
1709  CGF.getProfileCount(&S));
1710  CGF.EmitBlock(ThenBlock);
1711  CGF.incrementProfileCounter(&S);
1712  }
1713 
1714  // Emit the loop iteration variable.
1715  const Expr *IVExpr = S.getIterationVariable();
1716  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1717  CGF.EmitVarDecl(*IVDecl);
1718  CGF.EmitIgnoredExpr(S.getInit());
1719 
1720  // Emit the iterations count variable.
1721  // If it is not a variable, Sema decided to calculate iterations count on
1722  // each iteration (e.g., it is foldable into a constant).
1723  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
1724  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1725  // Emit calculation of the iterations count.
1727  }
1728 
1729  CGF.EmitOMPSimdInit(S);
1730 
1731  emitAlignedClause(CGF, S);
1732  (void)CGF.EmitOMPLinearClauseInit(S);
1733  {
1734  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
1735  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
1736  CGF.EmitOMPLinearClause(S, LoopScope);
1737  CGF.EmitOMPPrivateClause(S, LoopScope);
1738  CGF.EmitOMPReductionClauseInit(S, LoopScope);
1739  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
1740  (void)LoopScope.Privatize();
1742  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
1743  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
1744  S.getInc(),
1745  [&S](CodeGenFunction &CGF) {
1747  CGF.EmitStopPoint(&S);
1748  },
1749  [](CodeGenFunction &) {});
1750  CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
1751  // Emit final copy of the lastprivate variables at the end of loops.
1752  if (HasLastprivateClause)
1753  CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
1754  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
1756  [](CodeGenFunction &) { return nullptr; });
1757  }
1758  CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
1759  // Emit: if (PreCond) - end.
1760  if (ContBlock) {
1761  CGF.EmitBranch(ContBlock);
1762  CGF.EmitBlock(ContBlock, true);
1763  }
1764 }
1765 
1767  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
1768  emitOMPSimdRegion(CGF, S, Action);
1769  };
1770  OMPLexicalScope Scope(*this, S, OMPD_unknown);
1771  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
1772 }
1773 
1774 void CodeGenFunction::EmitOMPOuterLoop(
1775  bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
1777  const CodeGenFunction::OMPLoopArguments &LoopArgs,
1778  const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
1779  const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
1780  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1781 
1782  const Expr *IVExpr = S.getIterationVariable();
1783  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1784  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1785 
1786  JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
1787 
1788  // Start the loop with a block that tests the condition.
1789  llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
1790  EmitBlock(CondBlock);
1791  const SourceRange R = S.getSourceRange();
1792  LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
1793  SourceLocToDebugLoc(R.getEnd()));
1794 
1795  llvm::Value *BoolCondVal = nullptr;
1796  if (!DynamicOrOrdered) {
1797  // UB = min(UB, GlobalUB) or
1798  // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
1799  // 'distribute parallel for')
1800  EmitIgnoredExpr(LoopArgs.EUB);
1801  // IV = LB
1802  EmitIgnoredExpr(LoopArgs.Init);
1803  // IV < UB
1804  BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
1805  } else {
1806  BoolCondVal =
1807  RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
1808  LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
1809  }
1810 
1811  // If there are any cleanups between here and the loop-exit scope,
1812  // create a block to stage a loop exit along.
1813  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1814  if (LoopScope.requiresCleanups())
1815  ExitBlock = createBasicBlock("omp.dispatch.cleanup");
1816 
1817  llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
1818  Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1819  if (ExitBlock != LoopExit.getBlock()) {
1820  EmitBlock(ExitBlock);
1821  EmitBranchThroughCleanup(LoopExit);
1822  }
1823  EmitBlock(LoopBody);
1824 
1825  // Emit "IV = LB" (in case of static schedule, we have already calculated new
1826  // LB for loop condition and emitted it above).
1827  if (DynamicOrOrdered)
1828  EmitIgnoredExpr(LoopArgs.Init);
1829 
1830  // Create a block for the increment.
1831  JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
1832  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1833 
1834  // Generate !llvm.loop.parallel metadata for loads and stores for loops
1835  // with dynamic/guided scheduling and without ordered clause.
1837  LoopStack.setParallel(!IsMonotonic);
1838  else
1839  EmitOMPSimdInit(S, IsMonotonic);
1840 
1841  SourceLocation Loc = S.getBeginLoc();
1842 
1843  // when 'distribute' is not combined with a 'for':
1844  // while (idx <= UB) { BODY; ++idx; }
1845  // when 'distribute' is combined with a 'for'
1846  // (e.g. 'distribute parallel for')
1847  // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
1848  EmitOMPInnerLoop(
1849  S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
1850  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
1851  CodeGenLoop(CGF, S, LoopExit);
1852  },
1853  [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
1854  CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
1855  });
1856 
1857  EmitBlock(Continue.getBlock());
1858  BreakContinueStack.pop_back();
1859  if (!DynamicOrOrdered) {
1860  // Emit "LB = LB + Stride", "UB = UB + Stride".
1861  EmitIgnoredExpr(LoopArgs.NextLB);
1862  EmitIgnoredExpr(LoopArgs.NextUB);
1863  }
1864 
1865  EmitBranch(CondBlock);
1866  LoopStack.pop();
1867  // Emit the fall-through block.
1868  EmitBlock(LoopExit.getBlock());
1869 
1870  // Tell the runtime we are done.
1871  auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
1872  if (!DynamicOrOrdered)
1873  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
1874  S.getDirectiveKind());
1875  };
1876  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
1877 }
1878 
1879 void CodeGenFunction::EmitOMPForOuterLoop(
1880  const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
1881  const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
1882  const OMPLoopArguments &LoopArgs,
1883  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
1884  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1885 
1886  // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
1887  const bool DynamicOrOrdered =
1888  Ordered || RT.isDynamic(ScheduleKind.Schedule);
1889 
1890  assert((Ordered ||
1891  !RT.isStaticNonchunked(ScheduleKind.Schedule,
1892  LoopArgs.Chunk != nullptr)) &&
1893  "static non-chunked schedule does not need outer loop");
1894 
1895  // Emit outer loop.
1896  //
1897  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1898  // When schedule(dynamic,chunk_size) is specified, the iterations are
1899  // distributed to threads in the team in chunks as the threads request them.
1900  // Each thread executes a chunk of iterations, then requests another chunk,
1901  // until no chunks remain to be distributed. Each chunk contains chunk_size
1902  // iterations, except for the last chunk to be distributed, which may have
1903  // fewer iterations. When no chunk_size is specified, it defaults to 1.
1904  //
1905  // When schedule(guided,chunk_size) is specified, the iterations are assigned
1906  // to threads in the team in chunks as the executing threads request them.
1907  // Each thread executes a chunk of iterations, then requests another chunk,
1908  // until no chunks remain to be assigned. For a chunk_size of 1, the size of
1909  // each chunk is proportional to the number of unassigned iterations divided
1910  // by the number of threads in the team, decreasing to 1. For a chunk_size
1911  // with value k (greater than 1), the size of each chunk is determined in the
1912  // same way, with the restriction that the chunks do not contain fewer than k
1913  // iterations (except for the last chunk to be assigned, which may have fewer
1914  // than k iterations).
1915  //
1916  // When schedule(auto) is specified, the decision regarding scheduling is
1917  // delegated to the compiler and/or runtime system. The programmer gives the
1918  // implementation the freedom to choose any possible mapping of iterations to
1919  // threads in the team.
1920  //
1921  // When schedule(runtime) is specified, the decision regarding scheduling is
1922  // deferred until run time, and the schedule and chunk size are taken from the
1923  // run-sched-var ICV. If the ICV is set to auto, the schedule is
1924  // implementation defined
1925  //
1926  // while(__kmpc_dispatch_next(&LB, &UB)) {
1927  // idx = LB;
1928  // while (idx <= UB) { BODY; ++idx;
1929  // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
1930  // } // inner loop
1931  // }
1932  //
1933  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
1934  // When schedule(static, chunk_size) is specified, iterations are divided into
1935  // chunks of size chunk_size, and the chunks are assigned to the threads in
1936  // the team in a round-robin fashion in the order of the thread number.
1937  //
1938  // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
1939  // while (idx <= UB) { BODY; ++idx; } // inner loop
1940  // LB = LB + ST;
1941  // UB = UB + ST;
1942  // }
1943  //
1944 
1945  const Expr *IVExpr = S.getIterationVariable();
1946  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
1947  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
1948 
1949  if (DynamicOrOrdered) {
1950  const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
1951  CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
1952  llvm::Value *LBVal = DispatchBounds.first;
1953  llvm::Value *UBVal = DispatchBounds.second;
1954  CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
1955  LoopArgs.Chunk};
1956  RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
1957  IVSigned, Ordered, DipatchRTInputValues);
1958  } else {
1959  CGOpenMPRuntime::StaticRTInput StaticInit(
1960  IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
1961  LoopArgs.ST, LoopArgs.Chunk);
1962  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
1963  ScheduleKind, StaticInit);
1964  }
1965 
1966  auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
1967  const unsigned IVSize,
1968  const bool IVSigned) {
1969  if (Ordered) {
1970  CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
1971  IVSigned);
1972  }
1973  };
1974 
1975  OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
1976  LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
1977  OuterLoopArgs.IncExpr = S.getInc();
1978  OuterLoopArgs.Init = S.getInit();
1979  OuterLoopArgs.Cond = S.getCond();
1980  OuterLoopArgs.NextLB = S.getNextLowerBound();
1981  OuterLoopArgs.NextUB = S.getNextUpperBound();
1982  EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
1983  emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
1984 }
1985 
1987  const unsigned IVSize, const bool IVSigned) {}
1988 
1989 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1990  OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
1991  OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
1992  const CodeGenLoopTy &CodeGenLoopContent) {
1993 
1994  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
1995 
1996  // Emit outer loop.
1997  // Same behavior as a OMPForOuterLoop, except that schedule cannot be
1998  // dynamic
1999  //
2000 
2001  const Expr *IVExpr = S.getIterationVariable();
2002  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2003  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2004 
2005  CGOpenMPRuntime::StaticRTInput StaticInit(
2006  IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
2007  LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
2008  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
2009 
2010  // for combined 'distribute' and 'for' the increment expression of distribute
2011  // is stored in DistInc. For 'distribute' alone, it is in Inc.
2012  Expr *IncExpr;
2014  IncExpr = S.getDistInc();
2015  else
2016  IncExpr = S.getInc();
2017 
2018  // this routine is shared by 'omp distribute parallel for' and
2019  // 'omp distribute': select the right EUB expression depending on the
2020  // directive
2021  OMPLoopArguments OuterLoopArgs;
2022  OuterLoopArgs.LB = LoopArgs.LB;
2023  OuterLoopArgs.UB = LoopArgs.UB;
2024  OuterLoopArgs.ST = LoopArgs.ST;
2025  OuterLoopArgs.IL = LoopArgs.IL;
2026  OuterLoopArgs.Chunk = LoopArgs.Chunk;
2027  OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2029  : S.getEnsureUpperBound();
2030  OuterLoopArgs.IncExpr = IncExpr;
2031  OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2032  ? S.getCombinedInit()
2033  : S.getInit();
2034  OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2035  ? S.getCombinedCond()
2036  : S.getCond();
2037  OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2039  : S.getNextLowerBound();
2040  OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
2042  : S.getNextUpperBound();
2043 
2044  EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
2045  LoopScope, OuterLoopArgs, CodeGenLoopContent,
2047 }
2048 
2049 static std::pair<LValue, LValue>
2051  const OMPExecutableDirective &S) {
2052  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2053  LValue LB =
2054  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2055  LValue UB =
2056  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2057 
2058  // When composing 'distribute' with 'for' (e.g. as in 'distribute
2059  // parallel for') we need to use the 'distribute'
2060  // chunk lower and upper bounds rather than the whole loop iteration
2061  // space. These are parameters to the outlined function for 'parallel'
2062  // and we copy the bounds of the previous schedule into the
2063  // the current ones.
2064  LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
2065  LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
2066  llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
2067  PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
2068  PrevLBVal = CGF.EmitScalarConversion(
2069  PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
2070  LS.getIterationVariable()->getType(),
2072  llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
2073  PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
2074  PrevUBVal = CGF.EmitScalarConversion(
2075  PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
2076  LS.getIterationVariable()->getType(),
2078 
2079  CGF.EmitStoreOfScalar(PrevLBVal, LB);
2080  CGF.EmitStoreOfScalar(PrevUBVal, UB);
2081 
2082  return {LB, UB};
2083 }
2084 
2085 /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
2086 /// we need to use the LB and UB expressions generated by the worksharing
2087 /// code generation support, whereas in non combined situations we would
2088 /// just emit 0 and the LastIteration expression
2089 /// This function is necessary due to the difference of the LB and UB
2090 /// types for the RT emission routines for 'for_static_init' and
2091 /// 'for_dispatch_init'
2092 static std::pair<llvm::Value *, llvm::Value *>
2094  const OMPExecutableDirective &S,
2095  Address LB, Address UB) {
2096  const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
2097  const Expr *IVExpr = LS.getIterationVariable();
2098  // when implementing a dynamic schedule for a 'for' combined with a
2099  // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
2100  // is not normalized as each team only executes its own assigned
2101  // distribute chunk
2102  QualType IteratorTy = IVExpr->getType();
2103  llvm::Value *LBVal =
2104  CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2105  llvm::Value *UBVal =
2106  CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
2107  return {LBVal, UBVal};
2108 }
2109 
2111  CodeGenFunction &CGF, const OMPExecutableDirective &S,
2112  llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
2113  const auto &Dir = cast<OMPLoopDirective>(S);
2114  LValue LB =
2115  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
2116  llvm::Value *LBCast = CGF.Builder.CreateIntCast(
2117  CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2118  CapturedVars.push_back(LBCast);
2119  LValue UB =
2120  CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
2121 
2122  llvm::Value *UBCast = CGF.Builder.CreateIntCast(
2123  CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
2124  CapturedVars.push_back(UBCast);
2125 }
2126 
2127 static void
2129  const OMPLoopDirective &S,
2131  auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
2132  PrePostActionTy &Action) {
2133  Action.Enter(CGF);
2134  bool HasCancel = false;
2136  if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
2137  HasCancel = D->hasCancel();
2138  else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
2139  HasCancel = D->hasCancel();
2140  else if (const auto *D =
2141  dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
2142  HasCancel = D->hasCancel();
2143  }
2145  HasCancel);
2149  };
2150 
2152  CGF, S,
2153  isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
2154  CGInlinedWorksharingLoop,
2156 }
2157 
2160  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2162  S.getDistInc());
2163  };
2164  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2165  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2166 }
2167 
2170  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2172  S.getDistInc());
2173  };
2174  OMPLexicalScope Scope(*this, S, OMPD_parallel);
2175  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
2176 }
2177 
2179  const OMPDistributeSimdDirective &S) {
2180  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2182  };
2183  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2184  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2185 }
2186 
2188  CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
2189  // Emit SPMD target parallel for region as a standalone region.
2190  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2191  emitOMPSimdRegion(CGF, S, Action);
2192  };
2193  llvm::Function *Fn;
2194  llvm::Constant *Addr;
2195  // Emit target region as a standalone region.
2196  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
2197  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
2198  assert(Fn && Addr && "Target device function emission failed.");
2199 }
2200 
2202  const OMPTargetSimdDirective &S) {
2203  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2204  emitOMPSimdRegion(CGF, S, Action);
2205  };
2206  emitCommonOMPTargetDirective(*this, S, CodeGen);
2207 }
2208 
2209 namespace {
2210  struct ScheduleKindModifiersTy {
2214  ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
2217  : Kind(Kind), M1(M1), M2(M2) {}
2218  };
2219 } // namespace
2220 
2222  const OMPLoopDirective &S, Expr *EUB,
2223  const CodeGenLoopBoundsTy &CodeGenLoopBounds,
2224  const CodeGenDispatchBoundsTy &CGDispatchBounds) {
2225  // Emit the loop iteration variable.
2226  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
2227  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
2228  EmitVarDecl(*IVDecl);
2229 
2230  // Emit the iterations count variable.
2231  // If it is not a variable, Sema decided to calculate iterations count on each
2232  // iteration (e.g., it is foldable into a constant).
2233  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
2234  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
2235  // Emit calculation of the iterations count.
2236  EmitIgnoredExpr(S.getCalcLastIteration());
2237  }
2238 
2239  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
2240 
2241  bool HasLastprivateClause;
2242  // Check pre-condition.
2243  {
2244  OMPLoopScope PreInitScope(*this, S);
2245  // Skip the entire loop if we don't meet the precondition.
2246  // If the condition constant folds and can be elided, avoid emitting the
2247  // whole loop.
2248  bool CondConstant;
2249  llvm::BasicBlock *ContBlock = nullptr;
2250  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
2251  if (!CondConstant)
2252  return false;
2253  } else {
2254  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
2255  ContBlock = createBasicBlock("omp.precond.end");
2256  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
2257  getProfileCount(&S));
2258  EmitBlock(ThenBlock);
2259  incrementProfileCounter(&S);
2260  }
2261 
2262  RunCleanupsScope DoacrossCleanupScope(*this);
2263  bool Ordered = false;
2264  if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
2265  if (OrderedClause->getNumForLoops())
2266  RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
2267  else
2268  Ordered = true;
2269  }
2270 
2271  llvm::DenseSet<const Expr *> EmittedFinals;
2272  emitAlignedClause(*this, S);
2273  bool HasLinears = EmitOMPLinearClauseInit(S);
2274  // Emit helper vars inits.
2275 
2276  std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
2277  LValue LB = Bounds.first;
2278  LValue UB = Bounds.second;
2279  LValue ST =
2280  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
2281  LValue IL =
2282  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
2283 
2284  // Emit 'then' code.
2285  {
2286  OMPPrivateScope LoopScope(*this);
2287  if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
2288  // Emit implicit barrier to synchronize threads and avoid data races on
2289  // initialization of firstprivate variables and post-update of
2290  // lastprivate variables.
2291  CGM.getOpenMPRuntime().emitBarrierCall(
2292  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2293  /*ForceSimpleCall=*/true);
2294  }
2295  EmitOMPPrivateClause(S, LoopScope);
2296  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
2297  EmitOMPReductionClauseInit(S, LoopScope);
2298  EmitOMPPrivateLoopCounters(S, LoopScope);
2299  EmitOMPLinearClause(S, LoopScope);
2300  (void)LoopScope.Privatize();
2302  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
2303 
2304  // Detect the loop schedule kind and chunk.
2305  const Expr *ChunkExpr = nullptr;
2306  OpenMPScheduleTy ScheduleKind;
2307  if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
2308  ScheduleKind.Schedule = C->getScheduleKind();
2309  ScheduleKind.M1 = C->getFirstScheduleModifier();
2310  ScheduleKind.M2 = C->getSecondScheduleModifier();
2311  ChunkExpr = C->getChunkSize();
2312  } else {
2313  // Default behaviour for schedule clause.
2314  CGM.getOpenMPRuntime().getDefaultScheduleAndChunk(
2315  *this, S, ScheduleKind.Schedule, ChunkExpr);
2316  }
2317  bool HasChunkSizeOne = false;
2318  llvm::Value *Chunk = nullptr;
2319  if (ChunkExpr) {
2320  Chunk = EmitScalarExpr(ChunkExpr);
2321  Chunk = EmitScalarConversion(Chunk, ChunkExpr->getType(),
2323  S.getBeginLoc());
2324  Expr::EvalResult Result;
2325  if (ChunkExpr->EvaluateAsInt(Result, getContext())) {
2326  llvm::APSInt EvaluatedChunk = Result.Val.getInt();
2327  HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);
2328  }
2329  }
2330  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
2331  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
2332  // OpenMP 4.5, 2.7.1 Loop Construct, Description.
2333  // If the static schedule kind is specified or if the ordered clause is
2334  // specified, and if no monotonic modifier is specified, the effect will
2335  // be as if the monotonic modifier was specified.
2336  bool StaticChunkedOne = RT.isStaticChunked(ScheduleKind.Schedule,
2337  /* Chunked */ Chunk != nullptr) && HasChunkSizeOne &&
2339  if ((RT.isStaticNonchunked(ScheduleKind.Schedule,
2340  /* Chunked */ Chunk != nullptr) ||
2341  StaticChunkedOne) &&
2342  !Ordered) {
2344  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
2345  // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
2346  // When no chunk_size is specified, the iteration space is divided into
2347  // chunks that are approximately equal in size, and at most one chunk is
2348  // distributed to each thread. Note that the size of the chunks is
2349  // unspecified in this case.
2350  CGOpenMPRuntime::StaticRTInput StaticInit(
2351  IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
2352  UB.getAddress(), ST.getAddress(),
2353  StaticChunkedOne ? Chunk : nullptr);
2354  RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
2355  ScheduleKind, StaticInit);
2356  JumpDest LoopExit =
2357  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
2358  // UB = min(UB, GlobalUB);
2359  if (!StaticChunkedOne)
2360  EmitIgnoredExpr(S.getEnsureUpperBound());
2361  // IV = LB;
2362  EmitIgnoredExpr(S.getInit());
2363  // For unchunked static schedule generate:
2364  //
2365  // while (idx <= UB) {
2366  // BODY;
2367  // ++idx;
2368  // }
2369  //
2370  // For static schedule with chunk one:
2371  //
2372  // while (IV <= PrevUB) {
2373  // BODY;
2374  // IV += ST;
2375  // }
2376  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
2377  StaticChunkedOne ? S.getCombinedParForInDistCond() : S.getCond(),
2378  StaticChunkedOne ? S.getDistInc() : S.getInc(),
2379  [&S, LoopExit](CodeGenFunction &CGF) {
2380  CGF.EmitOMPLoopBody(S, LoopExit);
2381  CGF.EmitStopPoint(&S);
2382  },
2383  [](CodeGenFunction &) {});
2384  EmitBlock(LoopExit.getBlock());
2385  // Tell the runtime we are done.
2386  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2387  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2388  S.getDirectiveKind());
2389  };
2390  OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
2391  } else {
2392  const bool IsMonotonic =
2393  Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
2394  ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
2395  ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2396  ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2397  // Emit the outer loop, which requests its work chunk [LB..UB] from
2398  // runtime and runs the inner loop to process it.
2399  const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
2400  ST.getAddress(), IL.getAddress(),
2401  Chunk, EUB);
2402  EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2403  LoopArguments, CGDispatchBounds);
2404  }
2406  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
2407  return CGF.Builder.CreateIsNotNull(
2408  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2409  });
2410  }
2411  EmitOMPReductionClauseFinal(
2412  S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
2413  ? /*Parallel and Simd*/ OMPD_parallel_for_simd
2414  : /*Parallel only*/ OMPD_parallel);
2415  // Emit post-update of the reduction variables if IsLastIter != 0.
2417  *this, S, [IL, &S](CodeGenFunction &CGF) {
2418  return CGF.Builder.CreateIsNotNull(
2419  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2420  });
2421  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2422  if (HasLastprivateClause)
2423  EmitOMPLastprivateClauseFinal(
2425  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
2426  }
2427  EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
2428  return CGF.Builder.CreateIsNotNull(
2429  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2430  });
2431  DoacrossCleanupScope.ForceCleanup();
2432  // We're now done with the loop, so jump to the continuation block.
2433  if (ContBlock) {
2434  EmitBranch(ContBlock);
2435  EmitBlock(ContBlock, /*IsFinished=*/true);
2436  }
2437  }
2438  return HasLastprivateClause;
2439 }
2440 
2441 /// The following two functions generate expressions for the loop lower
2442 /// and upper bounds in case of static and dynamic (dispatch) schedule
2443 /// of the associated 'for' or 'distribute' loop.
2444 static std::pair<LValue, LValue>
2446  const auto &LS = cast<OMPLoopDirective>(S);
2447  LValue LB =
2448  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
2449  LValue UB =
2450  EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
2451  return {LB, UB};
2452 }
2453 
2454 /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
2455 /// consider the lower and upper bound expressions generated by the
2456 /// worksharing loop support, but we use 0 and the iteration space size as
2457 /// constants
2458 static std::pair<llvm::Value *, llvm::Value *>
2460  Address LB, Address UB) {
2461  const auto &LS = cast<OMPLoopDirective>(S);
2462  const Expr *IVExpr = LS.getIterationVariable();
2463  const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
2464  llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
2465  llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
2466  return {LBVal, UBVal};
2467 }
2468 
2470  bool HasLastprivates = false;
2471  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2472  PrePostActionTy &) {
2473  OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
2474  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2477  };
2478  {
2479  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2480  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
2481  S.hasCancel());
2482  }
2483 
2484  // Emit an implicit barrier at the end.
2485  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2486  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2487 }
2488 
2490  bool HasLastprivates = false;
2491  auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
2492  PrePostActionTy &) {
2493  HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
2496  };
2497  {
2498  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2499  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
2500  }
2501 
2502  // Emit an implicit barrier at the end.
2503  if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
2504  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
2505 }
2506 
2508  const Twine &Name,
2509  llvm::Value *Init = nullptr) {
2510  LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
2511  if (Init)
2512  CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
2513  return LVal;
2514 }
2515 
2516 void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
2518  const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
2519  bool HasLastprivates = false;
2520  auto &&CodeGen = [&S, CapturedStmt, CS,
2521  &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
2522  ASTContext &C = CGF.getContext();
2523  QualType KmpInt32Ty =
2524  C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2525  // Emit helper vars inits.
2526  LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
2527  CGF.Builder.getInt32(0));
2528  llvm::ConstantInt *GlobalUBVal = CS != nullptr
2529  ? CGF.Builder.getInt32(CS->size() - 1)
2530  : CGF.Builder.getInt32(0);
2531  LValue UB =
2532  createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
2533  LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
2534  CGF.Builder.getInt32(1));
2535  LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
2536  CGF.Builder.getInt32(0));
2537  // Loop counter.
2538  LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
2539  OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2540  CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
2541  OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
2542  CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
2543  // Generate condition for loop.
2544  BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
2546  // Increment for loop counter.
2547  UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
2548  S.getBeginLoc(), true);
2549  auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
2550  // Iterate through all sections and emit a switch construct:
2551  // switch (IV) {
2552  // case 0:
2553  // <SectionStmt[0]>;
2554  // break;
2555  // ...
2556  // case <NumSection> - 1:
2557  // <SectionStmt[<NumSection> - 1]>;
2558  // break;
2559  // }
2560  // .omp.sections.exit:
2561  llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
2562  llvm::SwitchInst *SwitchStmt =
2563  CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
2564  ExitBB, CS == nullptr ? 1 : CS->size());
2565  if (CS) {
2566  unsigned CaseNumber = 0;
2567  for (const Stmt *SubStmt : CS->children()) {
2568  auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
2569  CGF.EmitBlock(CaseBB);
2570  SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2571  CGF.EmitStmt(SubStmt);
2572  CGF.EmitBranch(ExitBB);
2573  ++CaseNumber;
2574  }
2575  } else {
2576  llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
2577  CGF.EmitBlock(CaseBB);
2578  SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2579  CGF.EmitStmt(CapturedStmt);
2580  CGF.EmitBranch(ExitBB);
2581  }
2582  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2583  };
2584 
2585  CodeGenFunction::OMPPrivateScope LoopScope(CGF);
2586  if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2587  // Emit implicit barrier to synchronize threads and avoid data races on
2588  // initialization of firstprivate variables and post-update of lastprivate
2589  // variables.
2590  CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2591  CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
2592  /*ForceSimpleCall=*/true);
2593  }
2594  CGF.EmitOMPPrivateClause(S, LoopScope);
2595  HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2596  CGF.EmitOMPReductionClauseInit(S, LoopScope);
2597  (void)LoopScope.Privatize();
2598  if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()))
2599  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
2600 
2601  // Emit static non-chunked loop.
2602  OpenMPScheduleTy ScheduleKind;
2603  ScheduleKind.Schedule = OMPC_SCHEDULE_static;
2604  CGOpenMPRuntime::StaticRTInput StaticInit(
2605  /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
2606  LB.getAddress(), UB.getAddress(), ST.getAddress());
2607  CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2608  CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
2609  // UB = min(UB, GlobalUB);
2610  llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
2611  llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
2612  CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2613  CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2614  // IV = LB;
2615  CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
2616  // while (idx <= UB) { BODY; ++idx; }
2617  CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
2618  [](CodeGenFunction &) {});
2619  // Tell the runtime we are done.
2620  auto &&CodeGen = [&S](CodeGenFunction &CGF) {
2621  CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
2622  S.getDirectiveKind());
2623  };
2624  CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
2625  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
2626  // Emit post-update of the reduction variables if IsLastIter != 0.
2627  emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
2628  return CGF.Builder.CreateIsNotNull(
2629  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
2630  });
2631 
2632  // Emit final copy of the lastprivate variables if IsLastIter != 0.
2633  if (HasLastprivates)
2635  S, /*NoFinals=*/false,
2636  CGF.Builder.CreateIsNotNull(
2637  CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
2638  };
2639 
2640  bool HasCancel = false;
2641  if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2642  HasCancel = OSD->hasCancel();
2643  else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2644  HasCancel = OPSD->hasCancel();
2645  OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
2646  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
2647  HasCancel);
2648  // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
2649  // clause. Otherwise the barrier will be generated by the codegen for the
2650  // directive.
2651  if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
2652  // Emit implicit barrier to synchronize threads and avoid data races on
2653  // initialization of firstprivate variables.
2654  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2655  OMPD_unknown);
2656  }
2657 }
2658 
2660  {
2661  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2662  EmitSections(S);
2663  }
2664  // Emit an implicit barrier at the end.
2665  if (!S.getSingleClause<OMPNowaitClause>()) {
2666  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
2667  OMPD_sections);
2668  }
2669 }
2670 
2672  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
2674  };
2675  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2676  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
2677  S.hasCancel());
2678 }
2679 
2681  llvm::SmallVector<const Expr *, 8> CopyprivateVars;
2684  llvm::SmallVector<const Expr *, 8> AssignmentOps;
2685  // Check if there are any 'copyprivate' clauses associated with this
2686  // 'single' construct.
2687  // Build a list of copyprivate variables along with helper expressions
2688  // (<source>, <destination>, <destination>=<source> expressions)
2689  for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
2690  CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2691  DestExprs.append(C->destination_exprs().begin(),
2692  C->destination_exprs().end());
2693  SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2694  AssignmentOps.append(C->assignment_ops().begin(),
2695  C->assignment_ops().end());
2696  }
2697  // Emit code for 'single' region along with 'copyprivate' clauses
2698  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2699  Action.Enter(CGF);
2700  OMPPrivateScope SingleScope(CGF);
2701  (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
2702  CGF.EmitOMPPrivateClause(S, SingleScope);
2703  (void)SingleScope.Privatize();
2705  };
2706  {
2707  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2708  CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
2709  CopyprivateVars, DestExprs,
2710  SrcExprs, AssignmentOps);
2711  }
2712  // Emit an implicit barrier at the end (to avoid data race on firstprivate
2713  // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
2714  if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
2715  CGM.getOpenMPRuntime().emitBarrierCall(
2716  *this, S.getBeginLoc(),
2717  S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
2718  }
2719 }
2720 
2722  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2723  Action.Enter(CGF);
2725  };
2726  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2727  CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
2728 }
2729 
2731  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2732  Action.Enter(CGF);
2734  };
2735  const Expr *Hint = nullptr;
2736  if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
2737  Hint = HintClause->getHint();
2738  OMPLexicalScope Scope(*this, S, OMPD_unknown);
2739  CGM.getOpenMPRuntime().emitCriticalRegion(*this,
2741  CodeGen, S.getBeginLoc(), Hint);
2742 }
2743 
2745  const OMPParallelForDirective &S) {
2746  // Emit directive as a combined directive that consists of two implicit
2747  // directives: 'parallel' with 'for' directive.
2748  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2749  Action.Enter(CGF);
2750  OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
2753  };
2754  emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
2756 }
2757 
2759  const OMPParallelForSimdDirective &S) {
2760  // Emit directive as a combined directive that consists of two implicit
2761  // directives: 'parallel' with 'for' directive.
2762  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2763  Action.Enter(CGF);
2766  };
2767  emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
2769 }
2770 
2772  const OMPParallelSectionsDirective &S) {
2773  // Emit directive as a combined directive that consists of two implicit
2774  // directives: 'parallel' with 'sections' directive.
2775  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
2776  Action.Enter(CGF);
2777  CGF.EmitSections(S);
2778  };
2779  emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
2781 }
2782 
2784  const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
2785  const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
2786  OMPTaskDataTy &Data) {
2787  // Emit outlined function for task construct.
2788  const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
2789  auto I = CS->getCapturedDecl()->param_begin();
2790  auto PartId = std::next(I);
2791  auto TaskT = std::next(I, 4);
2792  // Check if the task is final
2793  if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
2794  // If the condition constant folds and can be elided, try to avoid emitting
2795  // the condition and the dead arm of the if/else.
2796  const Expr *Cond = Clause->getCondition();
2797  bool CondConstant;
2798  if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
2799  Data.Final.setInt(CondConstant);
2800  else
2801  Data.Final.setPointer(EvaluateExprAsBool(Cond));
2802  } else {
2803  // By default the task is not final.
2804  Data.Final.setInt(/*IntVal=*/false);
2805  }
2806  // Check if the task has 'priority' clause.
2807  if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
2808  const Expr *Prio = Clause->getPriority();
2809  Data.Priority.setInt(/*IntVal=*/true);
2810  Data.Priority.setPointer(EmitScalarConversion(
2811  EmitScalarExpr(Prio), Prio->getType(),
2812  getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
2813  Prio->getExprLoc()));
2814  }
2815  // The first function argument for tasks is a thread id, the second one is a
2816  // part id (0 for tied tasks, >=0 for untied task).
2817  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
2818  // Get list of private variables.
2819  for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
2820  auto IRef = C->varlist_begin();
2821  for (const Expr *IInit : C->private_copies()) {
2822  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2823  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2824  Data.PrivateVars.push_back(*IRef);
2825  Data.PrivateCopies.push_back(IInit);
2826  }
2827  ++IRef;
2828  }
2829  }
2830  EmittedAsPrivate.clear();
2831  // Get list of firstprivate variables.
2832  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
2833  auto IRef = C->varlist_begin();
2834  auto IElemInitRef = C->inits().begin();
2835  for (const Expr *IInit : C->private_copies()) {
2836  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2837  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2838  Data.FirstprivateVars.push_back(*IRef);
2839  Data.FirstprivateCopies.push_back(IInit);
2840  Data.FirstprivateInits.push_back(*IElemInitRef);
2841  }
2842  ++IRef;
2843  ++IElemInitRef;
2844  }
2845  }
2846  // Get list of lastprivate variables (for taskloops).
2847  llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2848  for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
2849  auto IRef = C->varlist_begin();
2850  auto ID = C->destination_exprs().begin();
2851  for (const Expr *IInit : C->private_copies()) {
2852  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2853  if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2854  Data.LastprivateVars.push_back(*IRef);
2855  Data.LastprivateCopies.push_back(IInit);
2856  }
2857  LastprivateDstsOrigs.insert(
2858  {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2859  cast<DeclRefExpr>(*IRef)});
2860  ++IRef;
2861  ++ID;
2862  }
2863  }
2866  for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
2867  auto IPriv = C->privates().begin();
2868  auto IRed = C->reduction_ops().begin();
2869  auto ILHS = C->lhs_exprs().begin();
2870  auto IRHS = C->rhs_exprs().begin();
2871  for (const Expr *Ref : C->varlists()) {
2872  Data.ReductionVars.emplace_back(Ref);
2873  Data.ReductionCopies.emplace_back(*IPriv);
2874  Data.ReductionOps.emplace_back(*IRed);
2875  LHSs.emplace_back(*ILHS);
2876  RHSs.emplace_back(*IRHS);
2877  std::advance(IPriv, 1);
2878  std::advance(IRed, 1);
2879  std::advance(ILHS, 1);
2880  std::advance(IRHS, 1);
2881  }
2882  }
2883  Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
2884  *this, S.getBeginLoc(), LHSs, RHSs, Data);
2885  // Build list of dependences.
2886  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
2887  for (const Expr *IRef : C->varlists())
2888  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
2889  auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
2890  CapturedRegion](CodeGenFunction &CGF,
2891  PrePostActionTy &Action) {
2892  // Set proper addresses for generated private copies.
2893  OMPPrivateScope Scope(CGF);
2894  if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2895  !Data.LastprivateVars.empty()) {
2896  enum { PrivatesParam = 2, CopyFnParam = 3 };
2897  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
2898  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
2899  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
2900  CS->getCapturedDecl()->getParam(PrivatesParam)));
2901  // Map privates.
2904  CallArgs.push_back(PrivatesPtr);
2905  for (const Expr *E : Data.PrivateVars) {
2906  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2907  Address PrivatePtr = CGF.CreateMemTemp(
2908  CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
2909  PrivatePtrs.emplace_back(VD, PrivatePtr);
2910  CallArgs.push_back(PrivatePtr.getPointer());
2911  }
2912  for (const Expr *E : Data.FirstprivateVars) {
2913  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2914  Address PrivatePtr =
2915  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2916  ".firstpriv.ptr.addr");
2917  PrivatePtrs.emplace_back(VD, PrivatePtr);
2918  CallArgs.push_back(PrivatePtr.getPointer());
2919  }
2920  for (const Expr *E : Data.LastprivateVars) {
2921  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
2922  Address PrivatePtr =
2923  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
2924  ".lastpriv.ptr.addr");
2925  PrivatePtrs.emplace_back(VD, PrivatePtr);
2926  CallArgs.push_back(PrivatePtr.getPointer());
2927  }
2928  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
2929  CopyFn, CallArgs);
2930  for (const auto &Pair : LastprivateDstsOrigs) {
2931  const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2932  DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(OrigVD),
2933  /*RefersToEnclosingVariableOrCapture=*/
2934  CGF.CapturedStmtInfo->lookup(OrigVD) != nullptr,
2935  Pair.second->getType(), VK_LValue,
2936  Pair.second->getExprLoc());
2937  Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
2938  return CGF.EmitLValue(&DRE).getAddress();
2939  });
2940  }
2941  for (const auto &Pair : PrivatePtrs) {
2942  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
2943  CGF.getContext().getDeclAlign(Pair.first));
2944  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
2945  }
2946  }
2947  if (Data.Reductions) {
2948  OMPLexicalScope LexScope(CGF, S, CapturedRegion);
2950  Data.ReductionOps);
2951  llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
2952  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
2953  for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
2954  RedCG.emitSharedLValue(CGF, Cnt);
2955  RedCG.emitAggregateType(CGF, Cnt);
2956  // FIXME: This must removed once the runtime library is fixed.
2957  // Emit required threadprivate variables for
2958  // initializer/combiner/finalizer.
2959  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
2960  RedCG, Cnt);
2961  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
2962  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
2963  Replacement =
2964  Address(CGF.EmitScalarConversion(
2965  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
2966  CGF.getContext().getPointerType(
2967  Data.ReductionCopies[Cnt]->getType()),
2968  Data.ReductionCopies[Cnt]->getExprLoc()),
2969  Replacement.getAlignment());
2970  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
2971  Scope.addPrivate(RedCG.getBaseDecl(Cnt),
2972  [Replacement]() { return Replacement; });
2973  }
2974  }
2975  // Privatize all private variables except for in_reduction items.
2976  (void)Scope.Privatize();
2977  SmallVector<const Expr *, 4> InRedVars;
2978  SmallVector<const Expr *, 4> InRedPrivs;
2980  SmallVector<const Expr *, 4> TaskgroupDescriptors;
2981  for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
2982  auto IPriv = C->privates().begin();
2983  auto IRed = C->reduction_ops().begin();
2984  auto ITD = C->taskgroup_descriptors().begin();
2985  for (const Expr *Ref : C->varlists()) {
2986  InRedVars.emplace_back(Ref);
2987  InRedPrivs.emplace_back(*IPriv);
2988  InRedOps.emplace_back(*IRed);
2989  TaskgroupDescriptors.emplace_back(*ITD);
2990  std::advance(IPriv, 1);
2991  std::advance(IRed, 1);
2992  std::advance(ITD, 1);
2993  }
2994  }
2995  // Privatize in_reduction items here, because taskgroup descriptors must be
2996  // privatized earlier.
2997  OMPPrivateScope InRedScope(CGF);
2998  if (!InRedVars.empty()) {
2999  ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
3000  for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
3001  RedCG.emitSharedLValue(CGF, Cnt);
3002  RedCG.emitAggregateType(CGF, Cnt);
3003  // The taskgroup descriptor variable is always implicit firstprivate and
3004  // privatized already during processing of the firstprivates.
3005  // FIXME: This must removed once the runtime library is fixed.
3006  // Emit required threadprivate variables for
3007  // initializer/combiner/finalizer.
3008  CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
3009  RedCG, Cnt);
3010  llvm::Value *ReductionsPtr =
3011  CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
3012  TaskgroupDescriptors[Cnt]->getExprLoc());
3013  Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
3014  CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
3015  Replacement = Address(
3016  CGF.EmitScalarConversion(
3017  Replacement.getPointer(), CGF.getContext().VoidPtrTy,
3018  CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
3019  InRedPrivs[Cnt]->getExprLoc()),
3020  Replacement.getAlignment());
3021  Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
3022  InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
3023  [Replacement]() { return Replacement; });
3024  }
3025  }
3026  (void)InRedScope.Privatize();
3027 
3028  Action.Enter(CGF);
3029  BodyGen(CGF);
3030  };
3031  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3032  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
3033  Data.NumberOfParts);
3034  OMPLexicalScope Scope(*this, S);
3035  TaskGen(*this, OutlinedFn, Data);
3036 }
3037 
3038 static ImplicitParamDecl *
3040  QualType Ty, CapturedDecl *CD,
3041  SourceLocation Loc) {
3042  auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3044  auto *OrigRef = DeclRefExpr::Create(
3045  C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
3046  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3047  auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
3049  auto *PrivateRef = DeclRefExpr::Create(
3050  C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
3051  /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
3052  QualType ElemType = C.getBaseElementType(Ty);
3053  auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
3055  auto *InitRef = DeclRefExpr::Create(
3056  C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
3057  /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
3058  PrivateVD->setInitStyle(VarDecl::CInit);
3059  PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
3060  InitRef, /*BasePath=*/nullptr,
3061  VK_RValue));
3062  Data.FirstprivateVars.emplace_back(OrigRef);
3063  Data.FirstprivateCopies.emplace_back(PrivateRef);
3064  Data.FirstprivateInits.emplace_back(InitRef);
3065  return OrigVD;
3066 }
3067 
3069  const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
3070  OMPTargetDataInfo &InputInfo) {
3071  // Emit outlined function for task construct.
3072  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3073  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3074  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3075  auto I = CS->getCapturedDecl()->param_begin();
3076  auto PartId = std::next(I);
3077  auto TaskT = std::next(I, 4);
3078  OMPTaskDataTy Data;
3079  // The task is not final.
3080  Data.Final.setInt(/*IntVal=*/false);
3081  // Get list of firstprivate variables.
3082  for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
3083  auto IRef = C->varlist_begin();
3084  auto IElemInitRef = C->inits().begin();
3085  for (auto *IInit : C->private_copies()) {
3086  Data.FirstprivateVars.push_back(*IRef);
3087  Data.FirstprivateCopies.push_back(IInit);
3088  Data.FirstprivateInits.push_back(*IElemInitRef);
3089  ++IRef;
3090  ++IElemInitRef;
3091  }
3092  }
3093  OMPPrivateScope TargetScope(*this);
3094  VarDecl *BPVD = nullptr;
3095  VarDecl *PVD = nullptr;
3096  VarDecl *SVD = nullptr;
3097  if (InputInfo.NumberOfTargetItems > 0) {
3098  auto *CD = CapturedDecl::Create(
3099  getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
3100  llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
3101  QualType BaseAndPointersType = getContext().getConstantArrayType(
3102  getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
3103  /*IndexTypeQuals=*/0);
3105  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3107  getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
3108  QualType SizesType = getContext().getConstantArrayType(
3109  getContext().getSizeType(), ArrSize, ArrayType::Normal,
3110  /*IndexTypeQuals=*/0);
3111  SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
3112  S.getBeginLoc());
3113  TargetScope.addPrivate(
3114  BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
3115  TargetScope.addPrivate(PVD,
3116  [&InputInfo]() { return InputInfo.PointersArray; });
3117  TargetScope.addPrivate(SVD,
3118  [&InputInfo]() { return InputInfo.SizesArray; });
3119  }
3120  (void)TargetScope.Privatize();
3121  // Build list of dependences.
3122  for (const auto *C : S.getClausesOfKind<OMPDependClause>())
3123  for (const Expr *IRef : C->varlists())
3124  Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
3125  auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
3126  &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
3127  // Set proper addresses for generated private copies.
3128  OMPPrivateScope Scope(CGF);
3129  if (!Data.FirstprivateVars.empty()) {
3130  enum { PrivatesParam = 2, CopyFnParam = 3 };
3131  llvm::Value *CopyFn = CGF.Builder.CreateLoad(
3132  CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
3133  llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
3134  CS->getCapturedDecl()->getParam(PrivatesParam)));
3135  // Map privates.
3138  CallArgs.push_back(PrivatesPtr);
3139  for (const Expr *E : Data.FirstprivateVars) {
3140  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3141  Address PrivatePtr =
3142  CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
3143  ".firstpriv.ptr.addr");
3144  PrivatePtrs.emplace_back(VD, PrivatePtr);
3145  CallArgs.push_back(PrivatePtr.getPointer());
3146  }
3147  CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3148  CopyFn, CallArgs);
3149  for (const auto &Pair : PrivatePtrs) {
3150  Address Replacement(CGF.Builder.CreateLoad(Pair.second),
3151  CGF.getContext().getDeclAlign(Pair.first));
3152  Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
3153  }
3154  }
3155  // Privatize all private variables except for in_reduction items.
3156  (void)Scope.Privatize();
3157  if (InputInfo.NumberOfTargetItems > 0) {
3158  InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
3159  CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
3160  InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
3161  CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
3162  InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
3163  CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
3164  }
3165 
3166  Action.Enter(CGF);
3167  OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
3168  BodyGen(CGF);
3169  };
3170  llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
3171  S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
3172  Data.NumberOfParts);
3173  llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
3174  IntegerLiteral IfCond(getContext(), TrueOrFalse,
3175  getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
3176  SourceLocation());
3177 
3178  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
3179  SharedsTy, CapturedStruct, &IfCond, Data);
3180 }
3181 
3183  // Emit outlined function for task construct.
3184  const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
3185  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
3186  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
3187  const Expr *IfCond = nullptr;
3188  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
3189  if (C->getNameModifier() == OMPD_unknown ||
3190  C->getNameModifier() == OMPD_task) {
3191  IfCond = C->getCondition();
3192  break;
3193  }
3194  }
3195 
3196  OMPTaskDataTy Data;
3197  // Check if we should emit tied or untied task.
3198  Data.Tied = !S.getSingleClause<OMPUntiedClause>();
3199  auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
3200  CGF.EmitStmt(CS->getCapturedStmt());
3201  };
3202  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
3203  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
3204  const OMPTaskDataTy &Data) {
3205  CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
3206  SharedsTy, CapturedStruct, IfCond,
3207  Data);
3208  };
3209  EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
3210 }
3211 
3213  const OMPTaskyieldDirective &S) {
3214  CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
3215 }
3216 
3218  CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
3219 }
3220 
3222  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
3223 }
3224 
3226  const OMPTaskgroupDirective &S) {
3227  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
3228  Action.Enter(CGF);
3229  if (const Expr *E = S.getReductionRef()) {
3232  OMPTaskDataTy Data;
3233  for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
3234  auto IPriv = C->privates().begin();
3235  auto IRed = C->reduction_ops().begin();
3236  auto ILHS = C->lhs_exprs().begin();
3237  auto IRHS = C->rhs_exprs().begin();
3238  for (const Expr *Ref : C->varlists()) {
3239  Data.ReductionVars.emplace_back(Ref);
3240  Data.ReductionCopies.emplace_back(*IPriv);
3241  Data.ReductionOps.emplace_back(*IRed);
3242  LHSs.emplace_back(*ILHS);
3243  RHSs.emplace_back(*IRHS);
3244  std::advance(IPriv, 1);
3245  std::advance(IRed, 1);
3246  std::advance(ILHS, 1);
3247  std::advance(IRHS, 1);
3248  }
3249  }
3250  llvm::Value *ReductionDesc =
3251  CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
3252  LHSs, RHSs, Data);
3253  const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
3254  CGF.EmitVarDecl(*VD);
3255  CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
3256  /*Volatile=*/false, E->getType());
3257  }
3259  };
3260  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3261  CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
3262 }
3263 
3265  CGM.getOpenMPRuntime().emitFlush(
3266  *this,
3267  [&S]() -> ArrayRef<const Expr *> {
3268  if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
3269  return llvm::makeArrayRef(FlushClause->varlist_begin(),
3270  FlushClause->varlist_end());
3271  return llvm::None;
3272  }(),
3273  S.getBeginLoc());
3274 }
3275 
3277  const CodeGenLoopTy &CodeGenLoop,
3278  Expr *IncExpr) {
3279  // Emit the loop iteration variable.
3280  const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
3281  const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
3282  EmitVarDecl(*IVDecl);
3283 
3284  // Emit the iterations count variable.
3285  // If it is not a variable, Sema decided to calculate iterations count on each
3286  // iteration (e.g., it is foldable into a constant).
3287  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
3288  EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3289  // Emit calculation of the iterations count.
3290  EmitIgnoredExpr(S.getCalcLastIteration());
3291  }
3292 
3293  CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
3294 
3295  bool HasLastprivateClause = false;
3296  // Check pre-condition.
3297  {
3298  OMPLoopScope PreInitScope(*this, S);
3299  // Skip the entire loop if we don't meet the precondition.
3300  // If the condition constant folds and can be elided, avoid emitting the
3301  // whole loop.
3302  bool CondConstant;
3303  llvm::BasicBlock *ContBlock = nullptr;
3304  if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
3305  if (!CondConstant)
3306  return;
3307  } else {
3308  llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
3309  ContBlock = createBasicBlock("omp.precond.end");
3310  emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
3311  getProfileCount(&S));
3312  EmitBlock(ThenBlock);
3313  incrementProfileCounter(&S);
3314  }
3315 
3316  emitAlignedClause(*this, S);
3317  // Emit 'then' code.
3318  {
3319  // Emit helper vars inits.
3320 
3321  LValue LB = EmitOMPHelperVar(
3322  *this, cast<DeclRefExpr>(
3325  : S.getLowerBoundVariable())));
3326  LValue UB = EmitOMPHelperVar(
3327  *this, cast<DeclRefExpr>(
3330  : S.getUpperBoundVariable())));
3331  LValue ST =
3332  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
3333  LValue IL =
3334  EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
3335 
3336  OMPPrivateScope LoopScope(*this);
3337  if (EmitOMPFirstprivateClause(S, LoopScope)) {
3338  // Emit implicit barrier to synchronize threads and avoid data races
3339  // on initialization of firstprivate variables and post-update of
3340  // lastprivate variables.
3341  CGM.getOpenMPRuntime().emitBarrierCall(
3342  *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
3343  /*ForceSimpleCall=*/true);
3344  }
3345  EmitOMPPrivateClause(S, LoopScope);
3349  EmitOMPReductionClauseInit(S, LoopScope);
3350  HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
3351  EmitOMPPrivateLoopCounters(S, LoopScope);
3352  (void)LoopScope.Privatize();
3354  CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(*this, S);
3355 
3356  // Detect the distribute schedule kind and chunk.
3357  llvm::Value *Chunk = nullptr;
3359  if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
3360  ScheduleKind = C->getDistScheduleKind();
3361  if (const Expr *Ch = C->getChunkSize()) {
3362  Chunk = EmitScalarExpr(Ch);
3363  Chunk = EmitScalarConversion(Chunk, Ch->getType(),
3365  S.getBeginLoc());
3366  }
3367  } else {
3368  // Default behaviour for dist_schedule clause.
3369  CGM.getOpenMPRuntime().getDefaultDistScheduleAndChunk(
3370  *this, S, ScheduleKind, Chunk);
3371  }
3372  const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
3373  const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
3374 
3375  // OpenMP [2.10.8, distribute Construct, Description]
3376  // If dist_schedule is specified, kind must be static. If specified,
3377  // iterations are divided into chunks of size chunk_size, chunks are
3378  // assigned to the teams of the league in a round-robin fashion in the
3379  // order of the team number. When no chunk_size is specified, the
3380  // iteration space is divided into chunks that are approximately equal
3381  // in size, and at most one chunk is distributed to each team of the
3382  // league. The size of the chunks is unspecified in this case.
3383  bool StaticChunked = RT.isStaticChunked(
3384  ScheduleKind, /* Chunked */ Chunk != nullptr) &&
3386  if (RT.isStaticNonchunked(ScheduleKind,
3387  /* Chunked */ Chunk != nullptr) ||
3388  StaticChunked) {
3390  EmitOMPSimdInit(S, /*IsMonotonic=*/true);
3391  CGOpenMPRuntime::StaticRTInput StaticInit(
3392  IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
3393  LB.getAddress(), UB.getAddress(), ST.getAddress(),
3394  StaticChunked ? Chunk : nullptr);
3395  RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
3396  StaticInit);
3397  JumpDest LoopExit =
3398  getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
3399  // UB = min(UB, GlobalUB);
3402  : S.getEnsureUpperBound());
3403  // IV = LB;
3405  ? S.getCombinedInit()
3406  : S.getInit());
3407 
3408  const Expr *Cond =
3410  ? S.getCombinedCond()
3411  : S.getCond();
3412 
3413  if (StaticChunked)
3414  Cond = S.getCombinedDistCond();
3415 
3416  // For static unchunked schedules generate:
3417  //
3418  // 1. For distribute alone, codegen
3419  // while (idx <= UB) {
3420  // BODY;
3421  // ++idx;
3422  // }
3423  //
3424  // 2. When combined with 'for' (e.g. as in 'distribute parallel for')
3425  // while (idx <= UB) {
3426  // <CodeGen rest of pragma>(LB, UB);
3427  // idx += ST;
3428  // }
3429  //
3430  // For static chunk one schedule generate:
3431  //
3432  // while (IV <= GlobalUB) {
3433  // <CodeGen rest of pragma>(LB, UB);
3434  // LB += ST;
3435  // UB += ST;
3436  // UB = min(UB, GlobalUB);
3437  // IV = LB;
3438  // }
3439  //
3440  EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
3441  [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
3442  CodeGenLoop(CGF, S, LoopExit);
3443  },
3444  [&S, StaticChunked](CodeGenFunction &CGF) {
3445  if (StaticChunked) {
3446  CGF.EmitIgnoredExpr(S.getCombinedNextLowerBound());
3447  CGF.EmitIgnoredExpr(S.getCombinedNextUpperBound());
3448  CGF.EmitIgnoredExpr(S.getCombinedEnsureUpperBound());
3449  CGF.EmitIgnoredExpr(S.getCombinedInit());
3450  }
3451  });
3452  EmitBlock(LoopExit.getBlock());
3453  // Tell the runtime we are done.
3454  RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
3455  } else {
3456  // Emit the outer loop, which requests its work chunk [LB..UB] from
3457  // runtime and runs the inner loop to process it.
3458  const OMPLoopArguments LoopArguments = {
3459  LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
3460  Chunk};
3461  EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
3462  CodeGenLoop);
3463  }
3465  EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
3466  return CGF.Builder.CreateIsNotNull(
3467  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3468  });
3469  }
3473  EmitOMPReductionClauseFinal(S, OMPD_simd);
3474  // Emit post-update of the reduction variables if IsLastIter != 0.
3476  *this, S, [IL, &S](CodeGenFunction &CGF) {
3477  return CGF.Builder.CreateIsNotNull(
3478  CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
3479  });
3480  }
3481  // Emit final copy of the lastprivate variables if IsLastIter != 0.
3482  if (HasLastprivateClause) {
3483  EmitOMPLastprivateClauseFinal(
3484  S, /*NoFinals=*/false,
3485  Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
3486  }
3487  }
3488 
3489  // We're now done with the loop, so jump to the continuation block.
3490  if (ContBlock) {
3491  EmitBranch(ContBlock);
3492  EmitBlock(ContBlock, true);
3493  }
3494  }
3495 }
3496 
3498  const OMPDistributeDirective &S) {
3499  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
3501  };
3502  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3503  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
3504 }
3505 
3506 static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
3507  const CapturedStmt *S) {
3508  CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
3510  CGF.CapturedStmtInfo = &CapStmtInfo;
3511  llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
3512  Fn->setDoesNotRecurse();
3513  return Fn;
3514 }
3515 
3517  if (S.hasClausesOfKind<OMPDependClause>()) {
3518  assert(!S.getAssociatedStmt() &&
3519  "No associated statement must be in ordered depend construct.");
3520  for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
3521  CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
3522  return;
3523  }
3524  const auto *C = S.getSingleClause<OMPSIMDClause>();
3525  auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
3526  PrePostActionTy &Action) {
3527  const CapturedStmt *CS = S.getInnermostCapturedStmt();
3528  if (C) {
3530  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
3531  llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
3532  CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
3533  OutlinedFn, CapturedVars);
3534  } else {
3535  Action.Enter(CGF);
3536  CGF.EmitStmt(CS->getCapturedStmt());
3537  }
3538  };
3539  OMPLexicalScope Scope(*this, S, OMPD_unknown);
3540  CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
3541 }
3542 
3544  QualType SrcType, QualType DestType,
3545  SourceLocation Loc) {
3546  assert(CGF.hasScalarEvaluationKind(DestType) &&
3547  "DestType must have scalar evaluation kind.");
3548  assert(!Val.isAggregate() && "Must be a scalar or complex.");
3549  return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
3550  DestType, Loc)
3552  Val.getComplexVal(), SrcType, DestType, Loc);
3553 }
3554 
3557  QualType DestType, SourceLocation Loc) {
3558  assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
3559  "DestType must have complex evaluation kind.");
3560  CodeGenFunction::ComplexPairTy ComplexVal;
3561  if (Val.isScalar()) {
3562  // Convert the input element to the element type of the complex.
3563  QualType DestElementType =
3564  DestType->castAs<ComplexType>()->getElementType();
3565  llvm::Value *ScalarVal = CGF.EmitScalarConversion(
3566  Val.getScalarVal(), SrcType, DestElementType, Loc);
3567  ComplexVal = CodeGenFunction::ComplexPairTy(
3568  ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
3569  } else {
3570  assert(Val.isComplex() && "Must be a scalar or complex.");
3571  QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
3572  QualType DestElementType =
3573  DestType->castAs<ComplexType>()->getElementType();
3574  ComplexVal.first = CGF.EmitScalarConversion(
3575  Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
3576  ComplexVal.second = CGF.EmitScalarConversion(
3577  Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
3578  }
3579  return ComplexVal;
3580 }
3581 
3582 static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
3583  LValue LVal, RValue RVal) {
3584  if (LVal.isGlobalReg()) {
3585  CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
3586  } else {
3587  CGF.EmitAtomicStore(RVal, LVal,
3588  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3589  : llvm::AtomicOrdering::Monotonic,
3590  LVal.isVolatile(), /*IsInit=*/false);
3591  }
3592 }
3593 
3595  QualType RValTy, SourceLocation Loc) {
3596  switch (getEvaluationKind(LVal.getType())) {
3597  case TEK_Scalar:
3598  EmitStoreThroughLValue(RValue::get(convertToScalarValue(
3599  *this, RVal, RValTy, LVal.getType(), Loc)),
3600  LVal);
3601  break;
3602  case TEK_Complex:
3603  EmitStoreOfComplex(
3604  convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
3605  /*isInit=*/false);
3606  break;
3607  case TEK_Aggregate:
3608  llvm_unreachable("Must be a scalar or complex.");
3609  }
3610 }
3611 
3612 static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
3613  const Expr *X, const Expr *V,
3614  SourceLocation Loc) {
3615  // v = x;
3616  assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
3617  assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
3618  LValue XLValue = CGF.EmitLValue(X);
3619  LValue VLValue = CGF.EmitLValue(V);
3620  RValue Res = XLValue.isGlobalReg()
3621  ? CGF.EmitLoadOfLValue(XLValue, Loc)
3622  : CGF.EmitAtomicLoad(
3623  XLValue, Loc,
3624  IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3625  : llvm::AtomicOrdering::Monotonic,
3626  XLValue.isVolatile());
3627  // OpenMP, 2.12.6, atomic Construct
3628  // Any atomic construct with a seq_cst clause forces the atomically
3629  // performed operation to include an implicit flush operation without a
3630  // list.
3631  if (IsSeqCst)
3632  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3633  CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
3634 }
3635 
3636 static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
3637  const Expr *X, const Expr *E,
3638  SourceLocation Loc) {
3639  // x = expr;
3640  assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
3641  emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
3642  // OpenMP, 2.12.6, atomic Construct
3643  // Any atomic construct with a seq_cst clause forces the atomically
3644  // performed operation to include an implicit flush operation without a
3645  // list.
3646  if (IsSeqCst)
3647  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3648 }
3649 
3650 static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
3651  RValue Update,
3652  BinaryOperatorKind BO,
3653  llvm::AtomicOrdering AO,
3654  bool IsXLHSInRHSPart) {
3655  ASTContext &Context = CGF.getContext();
3656  // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
3657  // expression is simple and atomic is allowed for the given type for the
3658  // target platform.
3659  if (BO == BO_Comma || !Update.isScalar() ||
3660  !Update.getScalarVal()->getType()->isIntegerTy() ||
3661  !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
3662  (Update.getScalarVal()->getType() !=
3663  X.getAddress().getElementType())) ||
3664  !X.getAddress().getElementType()->isIntegerTy() ||
3665  !Context.getTargetInfo().hasBuiltinAtomic(
3666  Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
3667  return std::make_pair(false, RValue::get(nullptr));
3668 
3669  llvm::AtomicRMWInst::BinOp RMWOp;
3670  switch (BO) {
3671  case BO_Add:
3672  RMWOp = llvm::AtomicRMWInst::Add;
3673  break;
3674  case BO_Sub:
3675  if (!IsXLHSInRHSPart)
3676  return std::make_pair(false, RValue::get(nullptr));
3677  RMWOp = llvm::AtomicRMWInst::Sub;
3678  break;
3679  case BO_And:
3680  RMWOp = llvm::AtomicRMWInst::And;
3681  break;
3682  case BO_Or:
3683  RMWOp = llvm::AtomicRMWInst::Or;
3684  break;
3685  case BO_Xor:
3686  RMWOp = llvm::AtomicRMWInst::Xor;
3687  break;
3688  case BO_LT:
3690  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
3691  : llvm::AtomicRMWInst::Max)
3692  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
3693  : llvm::AtomicRMWInst::UMax);
3694  break;
3695  case BO_GT:
3697  ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
3698  : llvm::AtomicRMWInst::Min)
3699  : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
3700  : llvm::AtomicRMWInst::UMin);
3701  break;
3702  case BO_Assign:
3703  RMWOp = llvm::AtomicRMWInst::Xchg;
3704  break;
3705  case BO_Mul:
3706  case BO_Div:
3707  case BO_Rem:
3708  case BO_Shl:
3709  case BO_Shr:
3710  case BO_LAnd:
3711  case BO_LOr:
3712  return std::make_pair(false, RValue::get(nullptr));
3713  case BO_PtrMemD:
3714  case BO_PtrMemI:
3715  case BO_LE:
3716  case BO_GE:
3717  case BO_EQ:
3718  case BO_NE:
3719  case BO_Cmp:
3720  case BO_AddAssign:
3721  case BO_SubAssign:
3722  case BO_AndAssign:
3723  case BO_OrAssign:
3724  case BO_XorAssign:
3725  case BO_MulAssign:
3726  case BO_DivAssign:
3727  case BO_RemAssign:
3728  case BO_ShlAssign:
3729  case BO_ShrAssign:
3730  case BO_Comma:
3731  llvm_unreachable("Unsupported atomic update operation");
3732  }
3733  llvm::Value *UpdateVal = Update.getScalarVal();
3734  if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
3735  UpdateVal = CGF.Builder.CreateIntCast(
3736  IC, X.getAddress().getElementType(),
3738  }
3739  llvm::Value *Res =
3740  CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
3741  return std::make_pair(true, RValue::get(Res));
3742 }
3743 
3745  LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
3746  llvm::AtomicOrdering AO, SourceLocation Loc,
3747  const llvm::function_ref<RValue(RValue)> CommonGen) {
3748  // Update expressions are allowed to have the following forms:
3749  // x binop= expr; -> xrval + expr;
3750  // x++, ++x -> xrval + 1;
3751  // x--, --x -> xrval - 1;
3752  // x = x binop expr; -> xrval binop expr
3753  // x = expr Op x; - > expr binop xrval;
3754  auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
3755  if (!Res.first) {
3756  if (X.isGlobalReg()) {
3757  // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
3758  // 'xrval'.
3759  EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
3760  } else {
3761  // Perform compare-and-swap procedure.
3762  EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
3763  }
3764  }
3765  return Res;
3766 }
3767 
3768 static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
3769  const Expr *X, const Expr *E,
3770  const Expr *UE, bool IsXLHSInRHSPart,
3771  SourceLocation Loc) {
3772  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3773  "Update expr in 'atomic update' must be a binary operator.");
3774  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3775  // Update expressions are allowed to have the following forms:
3776  // x binop= expr; -> xrval + expr;
3777  // x++, ++x -> xrval + 1;
3778  // x--, --x -> xrval - 1;
3779  // x = x binop expr; -> xrval binop expr
3780  // x = expr Op x; - > expr binop xrval;
3781  assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
3782  LValue XLValue = CGF.EmitLValue(X);
3783  RValue ExprRValue = CGF.EmitAnyExpr(E);
3784  llvm::AtomicOrdering AO = IsSeqCst
3785  ? llvm::AtomicOrdering::SequentiallyConsistent
3786  : llvm::AtomicOrdering::Monotonic;
3787  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3788  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3789  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3790  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3791  auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
3792  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3793  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3794  return CGF.EmitAnyExpr(UE);
3795  };
3797  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3798  // OpenMP, 2.12.6, atomic Construct
3799  // Any atomic construct with a seq_cst clause forces the atomically
3800  // performed operation to include an implicit flush operation without a
3801  // list.
3802  if (IsSeqCst)
3803  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3804 }
3805 
3807  QualType SourceType, QualType ResType,
3808  SourceLocation Loc) {
3809  switch (CGF.getEvaluationKind(ResType)) {
3810  case TEK_Scalar:
3811  return RValue::get(
3812  convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
3813  case TEK_Complex: {
3814  auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
3815  return RValue::getComplex(Res.first, Res.second);
3816  }
3817  case TEK_Aggregate:
3818  break;
3819  }
3820  llvm_unreachable("Must be a scalar or complex.");
3821 }
3822 
3823 static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
3824  bool IsPostfixUpdate, const Expr *V,
3825  const Expr *X, const Expr *E,
3826  const Expr *UE, bool IsXLHSInRHSPart,
3827  SourceLocation Loc) {
3828  assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
3829  assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
3830  RValue NewVVal;
3831  LValue VLValue = CGF.EmitLValue(V);
3832  LValue XLValue = CGF.EmitLValue(X);
3833  RValue ExprRValue = CGF.EmitAnyExpr(E);
3834  llvm::AtomicOrdering AO = IsSeqCst
3835  ? llvm::AtomicOrdering::SequentiallyConsistent
3836  : llvm::AtomicOrdering::Monotonic;
3837  QualType NewVValType;
3838  if (UE) {
3839  // 'x' is updated with some additional value.
3840  assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
3841  "Update expr in 'atomic capture' must be a binary operator.");
3842  const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
3843  // Update expressions are allowed to have the following forms:
3844  // x binop= expr; -> xrval + expr;
3845  // x++, ++x -> xrval + 1;
3846  // x--, --x -> xrval - 1;
3847  // x = x binop expr; -> xrval binop expr
3848  // x = expr Op x; - > expr binop xrval;
3849  const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3850  const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3851  const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3852  NewVValType = XRValExpr->getType();
3853  const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3854  auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3855  IsPostfixUpdate](RValue XRValue) {
3856  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3857  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
3858  RValue Res = CGF.EmitAnyExpr(UE);
3859  NewVVal = IsPostfixUpdate ? XRValue : Res;
3860  return Res;
3861  };
3862  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3863  XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3864  if (Res.first) {
3865  // 'atomicrmw' instruction was generated.
3866  if (IsPostfixUpdate) {
3867  // Use old value from 'atomicrmw'.
3868  NewVVal = Res.second;
3869  } else {
3870  // 'atomicrmw' does not provide new value, so evaluate it using old
3871  // value of 'x'.
3872  CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
3873  CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
3874  NewVVal = CGF.EmitAnyExpr(UE);
3875  }
3876  }
3877  } else {
3878  // 'x' is simply rewritten with some 'expr'.
3879  NewVValType = X->getType().getNonReferenceType();
3880  ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
3881  X->getType().getNonReferenceType(), Loc);
3882  auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
3883  NewVVal = XRValue;
3884  return ExprRValue;
3885  };
3886  // Try to perform atomicrmw xchg, otherwise simple exchange.
3887  auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
3888  XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
3889  Loc, Gen);
3890  if (Res.first) {
3891  // 'atomicrmw' instruction was generated.
3892  NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3893  }
3894  }
3895  // Emit post-update store to 'v' of old/new 'x' value.
3896  CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
3897  // OpenMP, 2.12.6, atomic Construct
3898  // Any atomic construct with a seq_cst clause forces the atomically
3899  // performed operation to include an implicit flush operation without a
3900  // list.
3901  if (IsSeqCst)
3902  CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
3903 }
3904 
3906  bool IsSeqCst, bool IsPostfixUpdate,
3907  const Expr *X, const Expr *V, const Expr *E,
3908  const Expr *UE, bool IsXLHSInRHSPart,
3909  SourceLocation Loc) {
3910  switch (Kind) {
3911  case OMPC_read:
3912  emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
3913  break;
3914  case OMPC_write:
3915  emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
3916  break;
3917  case OMPC_unknown:
3918  case OMPC_update:
3919  emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
3920  break;
3921  case OMPC_capture:
3922  emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
3923  IsXLHSInRHSPart, Loc);
3924  break;
3925  case OMPC_if:
3926  case OMPC_final:
3927  case OMPC_num_threads:
3928  case OMPC_private:
3929  case OMPC_firstprivate:
3930  case OMPC_lastprivate:
3931  case OMPC_reduction:
3932  case OMPC_task_reduction:
3933  case OMPC_in_reduction:
3934  case OMPC_safelen:
3935  case OMPC_simdlen:
3936  case OMPC_collapse:
3937  case OMPC_default:
3938  case OMPC_seq_cst:
3939  case OMPC_shared:
3940  case OMPC_linear:
3941  case OMPC_aligned:
3942  case OMPC_copyin:
3943  case OMPC_copyprivate:
3944  case OMPC_flush:
3945  case OMPC_proc_bind:
3946  case OMPC_schedule:
3947  case OMPC_ordered:
3948  case OMPC_nowait:
3949  case OMPC_untied:
3950  case OMPC_threadprivate:
3951  case OMPC_depend:
3952  case OMPC_mergeable:
3953  case OMPC_device:
3954  case OMPC_threads:
3955  case OMPC_simd:
3956  case OMPC_map:
3957  case OMPC_num_teams:
3958  case OMPC_thread_limit:
3959  case OMPC_priority:
3960  case OMPC_grainsize:
3961  case OMPC_nogroup:
3962  case OMPC_num_tasks:
3963  case OMPC_hint:
3964  case OMPC_dist_schedule:
3965  case OMPC_defaultmap:
3966  case OMPC_uniform:
3967  case OMPC_to:
3968  case OMPC_from:
3969  case OMPC_use_device_ptr:
3970  case OMPC_is_device_ptr:
3971  case OMPC_unified_address:
3972  case OMPC_unified_shared_memory:
3973  case OMPC_reverse_offload:
3974  case OMPC_dynamic_allocators:
3975  case OMPC_atomic_default_mem_order:
3976  llvm_unreachable("Clause is not allowed in 'omp atomic'.");
3977  }
3978 }
3979 
3981  bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
3983  for (const OMPClause *C : S.clauses()) {
3984  // Find first clause (skip seq_cst clause, if it is first).
3985  if (C->getClauseKind() != OMPC_seq_cst) {
3986  Kind = C->getClauseKind();
3987  break;
3988  }
3989  }
3990 
3991  const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
3992  if (const auto *FE = dyn_cast<FullExpr>(CS))
3993  enterFullExpression(FE);
3994  // Processing for statements under 'atomic capture'.
3995  if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3996  for (const Stmt *C : Compound->body()) {
3997  if (const auto *FE = dyn_cast<FullExpr>(C))
3998  enterFullExpression(FE);
3999  }
4000  }
4001 
4002  auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
4003  PrePostActionTy &) {
4004  CGF.EmitStopPoint(CS);
4005  emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
4006  S.getV(), S.getExpr(), S.getUpdateExpr(),
4007  S.isXLHSInRHSPart(), S.getBeginLoc());
4008  };
4009  OMPLexicalScope Scope(*this, S, OMPD_unknown);
4010  CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
4011 }
4012 
4014  const OMPExecutableDirective &S,
4015  const RegionCodeGenTy &CodeGen) {
4017  CodeGenModule &CGM = CGF.CGM;
4018 
4019  // On device emit this construct as inlined code.
4020  if (CGM.getLangOpts().OpenMPIsDevice) {
4021  OMPLexicalScope Scope(CGF, S, OMPD_target);
4022  CGM.getOpenMPRuntime().emitInlinedDirective(
4023  CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4025  });
4026  return;
4027  }
4028 
4029  llvm::Function *Fn = nullptr;
4030  llvm::Constant *FnID = nullptr;
4031 
4032  const Expr *IfCond = nullptr;
4033  // Check for the at most one if clause associated with the target region.
4034  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4035  if (C->getNameModifier() == OMPD_unknown ||
4036  C->getNameModifier() == OMPD_target) {
4037  IfCond = C->getCondition();
4038  break;
4039  }
4040  }
4041 
4042  // Check if we have any device clause associated with the directive.
4043  const Expr *Device = nullptr;
4044  if (auto *C = S.getSingleClause<OMPDeviceClause>())
4045  Device = C->getDevice();
4046 
4047  // Check if we have an if clause whose conditional always evaluates to false
4048  // or if we do not have any targets specified. If so the target region is not
4049  // an offload entry point.
4050  bool IsOffloadEntry = true;
4051  if (IfCond) {
4052  bool Val;
4053  if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
4054  IsOffloadEntry = false;
4055  }
4056  if (CGM.getLangOpts().OMPTargetTriples.empty())
4057  IsOffloadEntry = false;
4058 
4059  assert(CGF.CurFuncDecl && "No parent declaration for target region!");
4060  StringRef ParentName;
4061  // In case we have Ctors/Dtors we use the complete type variant to produce
4062  // the mangling of the device outlined kernel.
4063  if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
4064  ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
4065  else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
4066  ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
4067  else
4068  ParentName =
4069  CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
4070 
4071  // Emit target region as a standalone region.
4072  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
4073  IsOffloadEntry, CodeGen);
4074  OMPLexicalScope Scope(CGF, S, OMPD_task);
4075  auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) {
4076  OMPLoopScope(CGF, D);
4077  // Emit calculation of the iterations count.
4078  llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
4079  NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
4080  /*IsSigned=*/false);
4081  return NumIterations;
4082  };
4083  CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device,
4084  SizeEmitter);
4085  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
4086 }
4087 
4089  PrePostActionTy &Action) {
4090  Action.Enter(CGF);
4091  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4092  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4093  CGF.EmitOMPPrivateClause(S, PrivateScope);
4094  (void)PrivateScope.Privatize();
4096  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4097 
4098  CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
4099 }
4100 
4102  StringRef ParentName,
4103  const OMPTargetDirective &S) {
4104  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4105  emitTargetRegion(CGF, S, Action);
4106  };
4107  llvm::Function *Fn;
4108  llvm::Constant *Addr;
4109  // Emit target region as a standalone region.
4110  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4111  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4112  assert(Fn && Addr && "Target device function emission failed.");
4113 }
4114 
4116  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4117  emitTargetRegion(CGF, S, Action);
4118  };
4119  emitCommonOMPTargetDirective(*this, S, CodeGen);
4120 }
4121 
4123  const OMPExecutableDirective &S,
4124  OpenMPDirectiveKind InnermostKind,
4125  const RegionCodeGenTy &CodeGen) {
4126  const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
4127  llvm::Value *OutlinedFn =
4128  CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
4129  S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
4130 
4131  const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
4132  const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
4133  if (NT || TL) {
4134  const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
4135  const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
4136 
4137  CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
4138  S.getBeginLoc());
4139  }
4140 
4141  OMPTeamsScope Scope(CGF, S);
4143  CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
4144  CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
4145  CapturedVars);
4146 }
4147 
4149  // Emit teams region as a standalone region.
4150  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4151  Action.Enter(CGF);
4152  OMPPrivateScope PrivateScope(CGF);
4153  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4154  CGF.EmitOMPPrivateClause(S, PrivateScope);
4155  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4156  (void)PrivateScope.Privatize();
4157  CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
4158  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4159  };
4160  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4162  [](CodeGenFunction &) { return nullptr; });
4163 }
4164 
4166  const OMPTargetTeamsDirective &S) {
4167  auto *CS = S.getCapturedStmt(OMPD_teams);
4168  Action.Enter(CGF);
4169  // Emit teams region as a standalone region.
4170  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4171  Action.Enter(CGF);
4172  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4173  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4174  CGF.EmitOMPPrivateClause(S, PrivateScope);
4175  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4176  (void)PrivateScope.Privatize();
4178  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4179  CGF.EmitStmt(CS->getCapturedStmt());
4180  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4181  };
4182  emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
4184  [](CodeGenFunction &) { return nullptr; });
4185 }
4186 
4188  CodeGenModule &CGM, StringRef ParentName,
4189  const OMPTargetTeamsDirective &S) {
4190  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4191  emitTargetTeamsRegion(CGF, Action, S);
4192  };
4193  llvm::Function *Fn;
4194  llvm::Constant *Addr;
4195  // Emit target region as a standalone region.
4196  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4197  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4198  assert(Fn && Addr && "Target device function emission failed.");
4199 }
4200 
4202  const OMPTargetTeamsDirective &S) {
4203  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4204  emitTargetTeamsRegion(CGF, Action, S);
4205  };
4206  emitCommonOMPTargetDirective(*this, S, CodeGen);
4207 }
4208 
4209 static void
4212  Action.Enter(CGF);
4213  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4215  };
4216 
4217  // Emit teams region as a standalone region.
4218  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4219  PrePostActionTy &Action) {
4220  Action.Enter(CGF);
4221  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4222  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4223  (void)PrivateScope.Privatize();
4224  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4225  CodeGenDistribute);
4226  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4227  };
4228  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
4230  [](CodeGenFunction &) { return nullptr; });
4231 }
4232 
4234  CodeGenModule &CGM, StringRef ParentName,
4236  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4237  emitTargetTeamsDistributeRegion(CGF, Action, S);
4238  };
4239  llvm::Function *Fn;
4240  llvm::Constant *Addr;
4241  // Emit target region as a standalone region.
4242  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4243  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4244  assert(Fn && Addr && "Target device function emission failed.");
4245 }
4246 
4249  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4250  emitTargetTeamsDistributeRegion(CGF, Action, S);
4251  };
4252  emitCommonOMPTargetDirective(*this, S, CodeGen);
4253 }
4254 
4256  CodeGenFunction &CGF, PrePostActionTy &Action,
4258  Action.Enter(CGF);
4259  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4261  };
4262 
4263  // Emit teams region as a standalone region.
4264  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4265  PrePostActionTy &Action) {
4266  Action.Enter(CGF);
4267  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4268  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4269  (void)PrivateScope.Privatize();
4270  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4271  CodeGenDistribute);
4272  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4273  };
4274  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
4276  [](CodeGenFunction &) { return nullptr; });
4277 }
4278 
4280  CodeGenModule &CGM, StringRef ParentName,
4282  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4283  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4284  };
4285  llvm::Function *Fn;
4286  llvm::Constant *Addr;
4287  // Emit target region as a standalone region.
4288  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4289  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4290  assert(Fn && Addr && "Target device function emission failed.");
4291 }
4292 
4295  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4296  emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
4297  };
4298  emitCommonOMPTargetDirective(*this, S, CodeGen);
4299 }
4300 
4302  const OMPTeamsDistributeDirective &S) {
4303 
4304  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4306  };
4307 
4308  // Emit teams region as a standalone region.
4309  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4310  PrePostActionTy &Action) {
4311  Action.Enter(CGF);
4312  OMPPrivateScope PrivateScope(CGF);
4313  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4314  (void)PrivateScope.Privatize();
4315  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4316  CodeGenDistribute);
4317  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4318  };
4319  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
4321  [](CodeGenFunction &) { return nullptr; });
4322 }
4323 
4326  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4328  };
4329 
4330  // Emit teams region as a standalone region.
4331  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4332  PrePostActionTy &Action) {
4333  Action.Enter(CGF);
4334  OMPPrivateScope PrivateScope(CGF);
4335  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4336  (void)PrivateScope.Privatize();
4337  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
4338  CodeGenDistribute);
4339  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4340  };
4341  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
4343  [](CodeGenFunction &) { return nullptr; });
4344 }
4345 
4348  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4350  S.getDistInc());
4351  };
4352 
4353  // Emit teams region as a standalone region.
4354  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4355  PrePostActionTy &Action) {
4356  Action.Enter(CGF);
4357  OMPPrivateScope PrivateScope(CGF);
4358  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4359  (void)PrivateScope.Privatize();
4360  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
4361  CodeGenDistribute);
4362  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4363  };
4364  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4366  [](CodeGenFunction &) { return nullptr; });
4367 }
4368 
4371  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4373  S.getDistInc());
4374  };
4375 
4376  // Emit teams region as a standalone region.
4377  auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4378  PrePostActionTy &Action) {
4379  Action.Enter(CGF);
4380  OMPPrivateScope PrivateScope(CGF);
4381  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4382  (void)PrivateScope.Privatize();
4383  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4384  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4385  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4386  };
4387  emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
4389  [](CodeGenFunction &) { return nullptr; });
4390 }
4391 
4394  PrePostActionTy &Action) {
4395  Action.Enter(CGF);
4396  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4398  S.getDistInc());
4399  };
4400 
4401  // Emit teams region as a standalone region.
4402  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4403  PrePostActionTy &Action) {
4404  Action.Enter(CGF);
4405  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4406  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4407  (void)PrivateScope.Privatize();
4408  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4409  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4410  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4411  };
4412 
4413  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
4414  CodeGenTeams);
4416  [](CodeGenFunction &) { return nullptr; });
4417 }
4418 
4420  CodeGenModule &CGM, StringRef ParentName,
4422  // Emit SPMD target teams distribute parallel for region as a standalone
4423  // region.
4424  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4426  };
4427  llvm::Function *Fn;
4428  llvm::Constant *Addr;
4429  // Emit target region as a standalone region.
4430  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4431  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4432  assert(Fn && Addr && "Target device function emission failed.");
4433 }
4434 
4437  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4439  };
4440  emitCommonOMPTargetDirective(*this, S, CodeGen);
4441 }
4442 
4444  CodeGenFunction &CGF,
4446  PrePostActionTy &Action) {
4447  Action.Enter(CGF);
4448  auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4450  S.getDistInc());
4451  };
4452 
4453  // Emit teams region as a standalone region.
4454  auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
4455  PrePostActionTy &Action) {
4456  Action.Enter(CGF);
4457  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4458  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4459  (void)PrivateScope.Privatize();
4460  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
4461  CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
4462  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
4463  };
4464 
4465  emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
4466  CodeGenTeams);
4468  [](CodeGenFunction &) { return nullptr; });
4469 }
4470 
4472  CodeGenModule &CGM, StringRef ParentName,
4474  // Emit SPMD target teams distribute parallel for simd region as a standalone
4475  // region.
4476  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4478  };
4479  llvm::Function *Fn;
4480  llvm::Constant *Addr;
4481  // Emit target region as a standalone region.
4482  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4483  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4484  assert(Fn && Addr && "Target device function emission failed.");
4485 }
4486 
4489  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4491  };
4492  emitCommonOMPTargetDirective(*this, S, CodeGen);
4493 }
4494 
4496  const OMPCancellationPointDirective &S) {
4497  CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
4498  S.getCancelRegion());
4499 }
4500 
4502  const Expr *IfCond = nullptr;
4503  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4504  if (C->getNameModifier() == OMPD_unknown ||
4505  C->getNameModifier() == OMPD_cancel) {
4506  IfCond = C->getCondition();
4507  break;
4508  }
4509  }
4510  CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
4511  S.getCancelRegion());
4512 }
4513 
4516  if (Kind == OMPD_parallel || Kind == OMPD_task ||
4517  Kind == OMPD_target_parallel)
4518  return ReturnBlock;
4519  assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
4520  Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
4521  Kind == OMPD_distribute_parallel_for ||
4522  Kind == OMPD_target_parallel_for ||
4523  Kind == OMPD_teams_distribute_parallel_for ||
4524  Kind == OMPD_target_teams_distribute_parallel_for);
4525  return OMPCancelStack.getExitBlock();
4526 }
4527 
4529  const OMPClause &NC, OMPPrivateScope &PrivateScope,
4530  const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
4531  const auto &C = cast<OMPUseDevicePtrClause>(NC);
4532  auto OrigVarIt = C.varlist_begin();
4533  auto InitIt = C.inits().begin();
4534  for (const Expr *PvtVarIt : C.private_copies()) {
4535  const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
4536  const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
4537  const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
4538 
4539  // In order to identify the right initializer we need to match the
4540  // declaration used by the mapping logic. In some cases we may get
4541  // OMPCapturedExprDecl that refers to the original declaration.
4542  const ValueDecl *MatchingVD = OrigVD;
4543  if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
4544  // OMPCapturedExprDecl are used to privative fields of the current
4545  // structure.
4546  const auto *ME = cast<MemberExpr>(OED->getInit());
4547  assert(isa<CXXThisExpr>(ME->getBase()) &&
4548  "Base should be the current struct!");
4549  MatchingVD = ME->getMemberDecl();
4550  }
4551 
4552  // If we don't have information about the current list item, move on to
4553  // the next one.
4554  auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
4555  if (InitAddrIt == CaptureDeviceAddrMap.end())
4556  continue;
4557 
4558  bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
4559  InitAddrIt, InitVD,
4560  PvtVD]() {
4561  // Initialize the temporary initialization variable with the address we
4562  // get from the runtime library. We have to cast the source address
4563  // because it is always a void *. References are materialized in the
4564  // privatization scope, so the initialization here disregards the fact
4565  // the original variable is a reference.
4566  QualType AddrQTy =
4567  getContext().getPointerType(OrigVD->getType().getNonReferenceType());
4568  llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
4569  Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
4570  setAddrOfLocalVar(InitVD, InitAddr);
4571 
4572  // Emit private declaration, it will be initialized by the value we
4573  // declaration we just added to the local declarations map.
4574  EmitDecl(*PvtVD);
4575 
4576  // The initialization variables reached its purpose in the emission
4577  // of the previous declaration, so we don't need it anymore.
4578  LocalDeclMap.erase(InitVD);
4579 
4580  // Return the address of the private variable.
4581  return GetAddrOfLocalVar(PvtVD);
4582  });
4583  assert(IsRegistered && "firstprivate var already registered as private");
4584  // Silence the warning about unused variable.
4585  (void)IsRegistered;
4586 
4587  ++OrigVarIt;
4588  ++InitIt;
4589  }
4590 }
4591 
4592 // Generate the instructions for '#pragma omp target data' directive.
4594  const OMPTargetDataDirective &S) {
4595  CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
4596 
4597  // Create a pre/post action to signal the privatization of the device pointer.
4598  // This action can be replaced by the OpenMP runtime code generation to
4599  // deactivate privatization.
4600  bool PrivatizeDevicePointers = false;
4601  class DevicePointerPrivActionTy : public PrePostActionTy {
4602  bool &PrivatizeDevicePointers;
4603 
4604  public:
4605  explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
4606  : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
4607  void Enter(CodeGenFunction &CGF) override {
4608  PrivatizeDevicePointers = true;
4609  }
4610  };
4611  DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
4612 
4613  auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
4614  CodeGenFunction &CGF, PrePostActionTy &Action) {
4615  auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
4617  };
4618 
4619  // Codegen that selects whether to generate the privatization code or not.
4620  auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
4621  &InnermostCodeGen](CodeGenFunction &CGF,
4622  PrePostActionTy &Action) {
4623  RegionCodeGenTy RCG(InnermostCodeGen);
4624  PrivatizeDevicePointers = false;
4625 
4626  // Call the pre-action to change the status of PrivatizeDevicePointers if
4627  // needed.
4628  Action.Enter(CGF);
4629 
4630  if (PrivatizeDevicePointers) {
4631  OMPPrivateScope PrivateScope(CGF);
4632  // Emit all instances of the use_device_ptr clause.
4633  for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
4634  CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
4635  Info.CaptureDeviceAddrMap);
4636  (void)PrivateScope.Privatize();
4637  RCG(CGF);
4638  } else {
4639  RCG(CGF);
4640  }
4641  };
4642 
4643  // Forward the provided action to the privatization codegen.
4644  RegionCodeGenTy PrivRCG(PrivCodeGen);
4645  PrivRCG.setAction(Action);
4646 
4647  // Notwithstanding the body of the region is emitted as inlined directive,
4648  // we don't use an inline scope as changes in the references inside the
4649  // region are expected to be visible outside, so we do not privative them.
4650  OMPLexicalScope Scope(CGF, S);
4651  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
4652  PrivRCG);
4653  };
4654 
4655  RegionCodeGenTy RCG(CodeGen);
4656 
4657  // If we don't have target devices, don't bother emitting the data mapping
4658  // code.
4659  if (CGM.getLangOpts().OMPTargetTriples.empty()) {
4660  RCG(*this);
4661  return;
4662  }
4663 
4664  // Check if we have any if clause associated with the directive.
4665  const Expr *IfCond = nullptr;
4666  if (const auto *C = S.getSingleClause<OMPIfClause>())
4667  IfCond = C->getCondition();
4668 
4669  // Check if we have any device clause associated with the directive.
4670  const Expr *Device = nullptr;
4671  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4672  Device = C->getDevice();
4673 
4674  // Set the action to signal privatization of device pointers.
4675  RCG.setAction(PrivAction);
4676 
4677  // Emit region code.
4678  CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
4679  Info);
4680 }
4681 
4683  const OMPTargetEnterDataDirective &S) {
4684  // If we don't have target devices, don't bother emitting the data mapping
4685  // code.
4686  if (CGM.getLangOpts().OMPTargetTriples.empty())
4687  return;
4688 
4689  // Check if we have any if clause associated with the directive.
4690  const Expr *IfCond = nullptr;
4691  if (const auto *C = S.getSingleClause<OMPIfClause>())
4692  IfCond = C->getCondition();
4693 
4694  // Check if we have any device clause associated with the directive.
4695  const Expr *Device = nullptr;
4696  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4697  Device = C->getDevice();
4698 
4699  OMPLexicalScope Scope(*this, S, OMPD_task);
4700  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4701 }
4702 
4704  const OMPTargetExitDataDirective &S) {
4705  // If we don't have target devices, don't bother emitting the data mapping
4706  // code.
4707  if (CGM.getLangOpts().OMPTargetTriples.empty())
4708  return;
4709 
4710  // Check if we have any if clause associated with the directive.
4711  const Expr *IfCond = nullptr;
4712  if (const auto *C = S.getSingleClause<OMPIfClause>())
4713  IfCond = C->getCondition();
4714 
4715  // Check if we have any device clause associated with the directive.
4716  const Expr *Device = nullptr;
4717  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
4718  Device = C->getDevice();
4719 
4720  OMPLexicalScope Scope(*this, S, OMPD_task);
4721  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
4722 }
4723 
4725  const OMPTargetParallelDirective &S,
4726  PrePostActionTy &Action) {
4727  // Get the captured statement associated with the 'parallel' region.
4728  const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
4729  Action.Enter(CGF);
4730  auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
4731  Action.Enter(CGF);
4732  CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
4733  (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
4734  CGF.EmitOMPPrivateClause(S, PrivateScope);
4735  CGF.EmitOMPReductionClauseInit(S, PrivateScope);
4736  (void)PrivateScope.Privatize();
4738  CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);
4739  // TODO: Add support for clauses.
4740  CGF.EmitStmt(CS->getCapturedStmt());
4741  CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
4742  };
4743  emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
4746  [](CodeGenFunction &) { return nullptr; });
4747 }
4748 
4750  CodeGenModule &CGM, StringRef ParentName,
4751  const OMPTargetParallelDirective &S) {
4752  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4753  emitTargetParallelRegion(CGF, S, Action);
4754  };
4755  llvm::Function *Fn;
4756  llvm::Constant *Addr;
4757  // Emit target region as a standalone region.
4758  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4759  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4760  assert(Fn && Addr && "Target device function emission failed.");
4761 }
4762 
4764  const OMPTargetParallelDirective &S) {
4765  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4766  emitTargetParallelRegion(CGF, S, Action);
4767  };
4768  emitCommonOMPTargetDirective(*this, S, CodeGen);
4769 }
4770 
4773  PrePostActionTy &Action) {
4774  Action.Enter(CGF);
4775  // Emit directive as a combined directive that consists of two implicit
4776  // directives: 'parallel' with 'for' directive.
4777  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4778  Action.Enter(CGF);
4780  CGF, OMPD_target_parallel_for, S.hasCancel());
4783  };
4784  emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
4786 }
4787 
4789  CodeGenModule &CGM, StringRef ParentName,
4790  const OMPTargetParallelForDirective &S) {
4791  // Emit SPMD target parallel for region as a standalone region.
4792  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4793  emitTargetParallelForRegion(CGF, S, Action);
4794  };
4795  llvm::Function *Fn;
4796  llvm::Constant *Addr;
4797  // Emit target region as a standalone region.
4798  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4799  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4800  assert(Fn && Addr && "Target device function emission failed.");
4801 }
4802 
4804  const OMPTargetParallelForDirective &S) {
4805  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4806  emitTargetParallelForRegion(CGF, S, Action);
4807  };
4808  emitCommonOMPTargetDirective(*this, S, CodeGen);
4809 }
4810 
4811 static void
4814  PrePostActionTy &Action) {
4815  Action.Enter(CGF);
4816  // Emit directive as a combined directive that consists of two implicit
4817  // directives: 'parallel' with 'for' directive.
4818  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4819  Action.Enter(CGF);
4822  };
4823  emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
4825 }
4826 
4828  CodeGenModule &CGM, StringRef ParentName,
4830  // Emit SPMD target parallel for region as a standalone region.
4831  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4832  emitTargetParallelForSimdRegion(CGF, S, Action);
4833  };
4834  llvm::Function *Fn;
4835  llvm::Constant *Addr;
4836  // Emit target region as a standalone region.
4837  CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
4838  S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
4839  assert(Fn && Addr && "Target device function emission failed.");
4840 }
4841 
4844  auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
4845  emitTargetParallelForSimdRegion(CGF, S, Action);
4846  };
4847  emitCommonOMPTargetDirective(*this, S, CodeGen);
4848 }
4849 
4850 /// Emit a helper variable and return corresponding lvalue.
4851 static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
4852  const ImplicitParamDecl *PVD,
4854  const auto *VDecl = cast<VarDecl>(Helper->getDecl());
4855  Privates.addPrivate(VDecl,
4856  [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
4857 }
4858 
4861  // Emit outlined function for task construct.
4862  const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
4863  Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
4864  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
4865  const Expr *IfCond = nullptr;
4866  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
4867  if (C->getNameModifier() == OMPD_unknown ||
4868  C->getNameModifier() == OMPD_taskloop) {
4869  IfCond = C->getCondition();
4870  break;
4871  }
4872  }
4873 
4874  OMPTaskDataTy Data;
4875  // Check if taskloop must be emitted without taskgroup.
4877  // TODO: Check if we should emit tied or untied task.
4878  Data.Tied = true;
4879  // Set scheduling for taskloop
4880  if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
4881  // grainsize clause
4882  Data.Schedule.setInt(/*IntVal=*/false);
4883  Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
4884  } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
4885  // num_tasks clause
4886  Data.Schedule.setInt(/*IntVal=*/true);
4887  Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
4888  }
4889 
4890  auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
4891  // if (PreCond) {
4892  // for (IV in 0..LastIteration) BODY;
4893  // <Final counter/linear vars updates>;
4894  // }
4895  //
4896 
4897  // Emit: if (PreCond) - begin.
4898  // If the condition constant folds and can be elided, avoid emitting the
4899  // whole loop.
4900  bool CondConstant;
4901  llvm::BasicBlock *ContBlock = nullptr;
4902  OMPLoopScope PreInitScope(CGF, S);
4903  if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
4904  if (!CondConstant)
4905  return;
4906  } else {
4907  llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
4908  ContBlock = CGF.createBasicBlock("taskloop.if.end");
4909  emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
4910  CGF.getProfileCount(&S));
4911  CGF.EmitBlock(ThenBlock);
4912  CGF.incrementProfileCounter(&S);
4913  }
4914 
4916  CGF.EmitOMPSimdInit(S);
4917 
4918  OMPPrivateScope LoopScope(CGF);
4919  // Emit helper vars inits.
4920  enum { LowerBound = 5, UpperBound, Stride, LastIter };
4921  auto *I = CS->getCapturedDecl()->param_begin();
4922  auto *LBP = std::next(I, LowerBound);
4923  auto *UBP = std::next(I, UpperBound);
4924  auto *STP = std::next(I, Stride);
4925  auto *LIP = std::next(I, LastIter);
4926  mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
4927  LoopScope);
4928  mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
4929  LoopScope);
4930  mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
4931  mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
4932  LoopScope);
4933  CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
4934  bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
4935  (void)LoopScope.Privatize();
4936  // Emit the loop iteration variable.
4937  const Expr *IVExpr = S.getIterationVariable();
4938  const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
4939  CGF.EmitVarDecl(*IVDecl);
4940  CGF.EmitIgnoredExpr(S.getInit());
4941 
4942  // Emit the iterations count variable.
4943  // If it is not a variable, Sema decided to calculate iterations count on
4944  // each iteration (e.g., it is foldable into a constant).
4945  if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
4946  CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
4947  // Emit calculation of the iterations count.
4948  CGF.EmitIgnoredExpr(S.getCalcLastIteration());
4949  }
4950 
4951  CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
4952  S.getInc(),
4953  [&S](CodeGenFunction &CGF) {
4954  CGF.EmitOMPLoopBody(S, JumpDest());
4955  CGF.EmitStopPoint(&S);
4956  },
4957  [](CodeGenFunction &) {});
4958  // Emit: if (PreCond) - end.
4959  if (ContBlock) {
4960  CGF.EmitBranch(ContBlock);
4961  CGF.EmitBlock(ContBlock, true);
4962  }
4963  // Emit final copy of the lastprivate variables if IsLastIter != 0.
4964  if (HasLastprivateClause) {
4965  CGF.EmitOMPLastprivateClauseFinal(
4967  CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
4968  CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
4969  (*LIP)->getType(), S.getBeginLoc())));
4970  }
4971  };
4972  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
4973  IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
4974  const OMPTaskDataTy &Data) {
4975  auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
4976  &Data](CodeGenFunction &CGF, PrePostActionTy &) {
4977  OMPLoopScope PreInitScope(CGF, S);
4978  CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
4979  OutlinedFn, SharedsTy,
4980  CapturedStruct, IfCond, Data);
4981  };
4982  CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
4983  CodeGen);
4984  };
4985  if (Data.Nogroup) {
4986  EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
4987  } else {
4988  CGM.getOpenMPRuntime().emitTaskgroupRegion(
4989  *this,
4990  [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
4991  PrePostActionTy &Action) {
4992  Action.Enter(CGF);
4993  CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
4994  Data);
4995  },
4996  S.getBeginLoc());
4997  }
4998 }
4999 
5001  EmitOMPTaskLoopBasedDirective(S);
5002 }
5003 
5005  const OMPTaskLoopSimdDirective &S) {
5006  EmitOMPTaskLoopBasedDirective(S);
5007 }
5008 
5009 // Generate the instructions for '#pragma omp target update' directive.
5011  const OMPTargetUpdateDirective &S) {
5012  // If we don't have target devices, don't bother emitting the data mapping
5013  // code.
5014  if (CGM.getLangOpts().OMPTargetTriples.empty())
5015  return;
5016 
5017  // Check if we have any if clause associated with the directive.
5018  const Expr *IfCond = nullptr;
5019  if (const auto *C = S.getSingleClause<OMPIfClause>())
5020  IfCond = C->getCondition();
5021 
5022  // Check if we have any device clause associated with the directive.
5023  const Expr *Device = nullptr;
5024  if (const auto *C = S.getSingleClause<OMPDeviceClause>())
5025  Device = C->getDevice();
5026 
5027  OMPLexicalScope Scope(*this, S, OMPD_task);
5028  CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
5029 }
5030 
5032  const OMPExecutableDirective &D) {
5033  if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
5034  return;
5035  auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
5037  emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
5038  } else {
5039  OMPPrivateScope LoopGlobals(CGF);
5040  if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
5041  for (const Expr *E : LD->counters()) {
5042  const auto *VD = dyn_cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
5043  if (!VD->hasLocalStorage() && !CGF.LocalDeclMap.count(VD)) {
5044  LValue GlobLVal = CGF.EmitLValue(E);
5045  LoopGlobals.addPrivate(
5046  VD, [&GlobLVal]() { return GlobLVal.getAddress(); });
5047  }
5048  if (isa<OMPCapturedExprDecl>(VD)) {
5049  // Emit only those that were not explicitly referenced in clauses.
5050  if (!CGF.LocalDeclMap.count(VD))
5051  CGF.EmitVarDecl(*VD);
5052  }
5053  }
5054  for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
5055  if (!C->getNumForLoops())
5056  continue;
5057  for (unsigned I = LD->getCollapsedNumber(),
5058  E = C->getLoopNumIterations().size();
5059  I < E; ++I) {
5060  if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
5061  cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
5062  // Emit only those that were not explicitly referenced in clauses.
5063  if (!CGF.LocalDeclMap.count(VD))
5064  CGF.EmitVarDecl(*VD);
5065  }
5066  }
5067  }
5068  }
5069  LoopGlobals.Privatize();
5071  }
5072  };
5073  OMPSimdLexicalScope Scope(*this, D);
5074  CGM.getOpenMPRuntime().emitInlinedDirective(
5075  *this,
5076  isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
5077  : D.getDirectiveKind(),
5078  CodeGen);
5079 }
5080 
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
Definition: CGCall.cpp:659
bool isAggregate() const
Definition: CGValue.h:54
This represents &#39;#pragma omp distribute simd&#39; composite directive.
Definition: StmtOpenMP.h:3248
Expr * getNextUpperBound() const
Definition: StmtOpenMP.h:865
This represents &#39;#pragma omp master&#39; directive.
Definition: StmtOpenMP.h:1431
This represents &#39;#pragma omp task&#39; directive.
Definition: StmtOpenMP.h:1771
static const Decl * getCanonicalDecl(const Decl *D)
Represents a function declaration or definition.
Definition: Decl.h:1738
This represents &#39;thread_limit&#39; clause in the &#39;#pragma omp ...&#39; directive.
Expr * getUpperBoundVariable() const
Definition: StmtOpenMP.h:833
Other implicit parameter.
Definition: Decl.h:1510
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, OMPTargetDataInfo &InputInfo)
static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
This represents clause &#39;copyin&#39; in the &#39;#pragma omp ...&#39; directives.
Complete object ctor.
Definition: ABI.h:26
PointerType - C99 6.7.5.1 - Pointer Declarators.
Definition: Type.h:2537
Scheduling data for loop-based OpenMP directives.
Definition: OpenMPKinds.h:141
A (possibly-)qualified type.
Definition: Type.h:638
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
Definition: Stmt.h:3279
bool isArrayType() const
Definition: Type.h:6345
ArrayRef< OMPClause * > clauses()
Definition: StmtOpenMP.h:260
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
Definition: CGDecl.cpp:158
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
Definition: Decl.h:4124
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
Definition: CGExpr.cpp:139
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
static void EmitOMPTargetParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForSimdDirective &S)
Emit device code for the target parallel for simd directive.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
Definition: Dominators.h:30
static std::pair< LValue, LValue > emitDistributeParallelForInnerBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
static OMPClauseWithPreInit * get(OMPClause *C)
Stmt - This represents one statement.
Definition: Stmt.h:66
This represents clause &#39;in_reduction&#39; in the &#39;#pragma omp task&#39; directives.
Expr * getLowerBoundVariable() const
Definition: StmtOpenMP.h:825
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
Definition: CGExpr.cpp:2149
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
Definition: Type.cpp:505
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S)
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents &#39;#pragma omp for simd&#39; directive.
Definition: StmtOpenMP.h:1181
SmallVector< const Expr *, 4 > LastprivateCopies
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
Decl - This represents one declaration (or definition), e.g.
Definition: DeclBase.h:87
This represents &#39;grainsize&#39; clause in the &#39;#pragma omp ...&#39; directive.
This represents &#39;#pragma omp teams distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3659
SourceLocation getBeginLoc() const
Returns starting location of directive kind.
Definition: StmtOpenMP.h:168
bool isNothrow() const
Definition: Decl.cpp:4469
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents &#39;if&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:240
CapturedStmt * getInnermostCapturedStmt()
Get innermost captured statement for the construct.
Definition: StmtOpenMP.h:226
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
Definition: Type.h:6245
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
bool isVolatile() const
Definition: CGValue.h:301
This represents &#39;priority&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
Definition: Type.h:1407
This represents &#39;#pragma omp target teams distribute&#39; combined directive.
Definition: StmtOpenMP.h:3796
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
Definition: CGExpr.cpp:1914
static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
Definition: Stmt.h:3223
Represents an array type, per C99 6.7.5.2 - Array Declarators.
Definition: Type.h:2812
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
const TargetInfo & getTargetInfo() const
Definition: ASTContext.h:690
Expr * getCombinedParForInDistCond() const
Definition: StmtOpenMP.h:953
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, const CodeGenBoundParametersTy &CodeGenBoundParameters)
Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
Definition: CGExpr.cpp:2322
Floating point control options.
Definition: LangOptions.h:307
This represents &#39;#pragma omp parallel for&#39; directive.
Definition: StmtOpenMP.h:1552
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value *> &CapturedVars)
This represents &#39;#pragma omp target teams distribute parallel for&#39; combined directive.
Definition: StmtOpenMP.h:3864
Expr * getCombinedEnsureUpperBound() const
Definition: StmtOpenMP.h:917
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
Definition: CGAtomic.cpp:1895
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
Represents a point when we exit a loop.
Definition: ProgramPoint.h:715
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
SmallVector< const Expr *, 4 > ReductionCopies
SourceLocation getEndLoc() const
Returns ending location of directive.
Definition: StmtOpenMP.h:170
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents &#39;#pragma omp target exit data&#39; directive.
Definition: StmtOpenMP.h:2463
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
CanQualType getCanonicalParamType(QualType T) const
Return the canonical parameter type corresponding to the specific potentially non-canonical one...
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
Represents a variable declaration or definition.
Definition: Decl.h:813
This represents clause &#39;private&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:2646
This represents &#39;num_threads&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:382
const Stmt * getBody() const
Definition: StmtOpenMP.h:959
Extra information about a function prototype.
Definition: Type.h:3767
uint64_t getProfileCount(const Stmt *S)
Get the profiler&#39;s count for the given statement.
Expr * getCombinedUpperBoundVariable() const
Definition: StmtOpenMP.h:911
Expr * getCalcLastIteration() const
Definition: StmtOpenMP.h:793
llvm::Value * getPointer() const
Definition: Address.h:38
unsigned getContextParamPosition() const
Definition: Decl.h:4118
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> BodyGen, const llvm::function_ref< void(CodeGenFunction &)> PostIncGen)
Emit inner loop of the worksharing/simd construct.
This represents implicit clause &#39;flush&#39; for the &#39;#pragma omp flush&#39; directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S)
static void emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeDirective &S)
bool isXLHSInRHSPart() const
Return true if helper update expression has form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; and...
Definition: StmtOpenMP.h:2260
static void EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelDirective &S)
This represents &#39;nogroup&#39; clause in the &#39;#pragma omp ...&#39; directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB, const CodeGenLoopBoundsTy &CodeGenLoopBounds, const CodeGenDispatchBoundsTy &CGDispatchBounds)
Emit code for the worksharing loop-based directive.
This represents &#39;safelen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:447
void EmitOMPTargetTeamsDistributeParallelForSimdDirective(const OMPTargetTeamsDistributeParallelForSimdDirective &S)
IdentifierInfo * getIdentifier() const
Get the identifier that names this declaration, if there is one.
Definition: Decl.h:270
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Definition: Expr.h:3167
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
Represents a struct/union/class.
Definition: Decl.h:3593
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents &#39;#pragma omp parallel&#39; directive.
Definition: StmtOpenMP.h:276
Address getAddress() const
Definition: CGValue.h:327
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:155
A C++ nested-name-specifier augmented with source location information.
This represents &#39;simd&#39; clause in the &#39;#pragma omp ...&#39; directive.
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
field_range fields() const
Definition: Decl.h:3784
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S, PrePostActionTy &Action)
Expr * getEnsureUpperBound() const
Definition: StmtOpenMP.h:849
SmallVector< const Expr *, 4 > LastprivateVars
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Stmt.cpp:288
Address EmitLoadOfReference(LValue RefLVal, LValueBaseInfo *PointeeBaseInfo=nullptr, TBAAAccessInfo *PointeeTBAAInfo=nullptr)
Definition: CGExpr.cpp:2300
Represents a member of a struct/union/class.
Definition: Decl.h:2579
This represents clause &#39;lastprivate&#39; in the &#39;#pragma omp ...&#39; directives.
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
void EmitOMPDistributeLoop(const OMPLoopDirective &S, const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr)
Emit code for the distribute loop-based directive.
CharUnits getAlignment() const
Definition: CGValue.h:316
const CapturedStmt * getCapturedStmt(OpenMPDirectiveKind RegionKind) const
Returns the captured statement associated with the component region within the (combined) directive...
Definition: StmtOpenMP.h:209
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
Expr * getInc() const
Definition: StmtOpenMP.h:809
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
Definition: StmtOpenMP.h:989
bool isReferenceType() const
Definition: Type.h:6308
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
static DeclRefExpr * Create(const ASTContext &Context, NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKWLoc, ValueDecl *D, bool RefersToEnclosingVariableOrCapture, SourceLocation NameLoc, QualType T, ExprValueKind VK, NamedDecl *FoundD=nullptr, const TemplateArgumentListInfo *TemplateArgs=nullptr)
Definition: Expr.cpp:406
This represents &#39;#pragma omp target simd&#39; directive.
Definition: StmtOpenMP.h:3384
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
OpenMPDirectiveKind getDirectiveKind() const
Definition: StmtOpenMP.h:244
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Decl.h:739
This represents &#39;#pragma omp barrier&#39; directive.
Definition: StmtOpenMP.h:1883
This is a common base class for loop directives (&#39;omp simd&#39;, &#39;omp for&#39;, &#39;omp for simd&#39; etc...
Definition: StmtOpenMP.h:338
This represents &#39;#pragma omp critical&#39; directive.
Definition: StmtOpenMP.h:1478
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2762
IdentifierTable & Idents
Definition: ASTContext.h:566
static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
Definition: Specifiers.h:110
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
Definition: CGExpr.cpp:194
This represents clause &#39;copyprivate&#39; in the &#39;#pragma omp ...&#39; directives.
OpenMPDistScheduleClauseKind
OpenMP attributes for &#39;dist_schedule&#39; clause.
Definition: OpenMPKinds.h:109
This represents &#39;#pragma omp distribute parallel for&#39; composite directive.
Definition: StmtOpenMP.h:3099
Stmt * getBody() const override
getBody - If this Decl represents a declaration for a body of code, such as a function or method defi...
Definition: Decl.cpp:4466
This represents &#39;#pragma omp teams distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3588
BinaryOperatorKind
static bool hasScalarEvaluationKind(QualType T)
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Definition: Stmt.h:3266
void EmitOMPTargetTeamsDistributeSimdDirective(const OMPTargetTeamsDistributeSimdDirective &S)
ArrayRef< Expr * > finals()
Definition: StmtOpenMP.h:995
Expr * getIsLastIterVariable() const
Definition: StmtOpenMP.h:817
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
OpenMPScheduleClauseKind Schedule
Definition: OpenMPKinds.h:142
CharUnits - This is an opaque type for sizes expressed in character units.
Definition: CharUnits.h:38
static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc, const unsigned IVSize, const bool IVSigned)
bool isOpenMPTeamsDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a teams-kind directive.
static void emitDistributeParallelForDistributeInnerBoundParams(CodeGenFunction &CGF, const OMPExecutableDirective &S, llvm::SmallVectorImpl< llvm::Value *> &CapturedVars)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
Definition: TargetInfo.h:614
llvm::function_ref< std::pair< LValue, LValue > CodeGenFunction &, const OMPExecutableDirective &S)> CodeGenLoopBoundsTy
Expr * getX()
Get &#39;x&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2244
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
Definition: Type.h:6797
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
Definition: Decl.h:4126
CharUnits getAlignment() const
Return the alignment of this pointer.
Definition: Address.h:67
void EmitOMPTeamsDistributeParallelForSimdDirective(const OMPTeamsDistributeParallelForSimdDirective &S)
void EmitOMPTargetDirective(const OMPTargetDirective &S)
Expr * getIterationVariable() const
Definition: StmtOpenMP.h:785
bool isComplex() const
Definition: CGValue.h:53
A builtin binary operation expression such as "x + y" or "x <= y".
Definition: Expr.h:3292
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
Definition: Type.h:6142
OpenMPScheduleClauseModifier M2
Definition: OpenMPKinds.h:144
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
Definition: CGLoopInfo.h:169
SmallVector< const Expr *, 4 > PrivateCopies
void restore(CodeGenFunction &CGF)
Restores original addresses of the variables.
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
This represents &#39;#pragma omp cancellation point&#39; directive.
Definition: StmtOpenMP.h:2718
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr)
Adjusts PrivatedAddr for using instead of the original variable address in normal operations...
Expr * getSizeExpr() const
Definition: Type.h:2991
Scope - A scope is a transient data structure that is used while parsing the program.
Definition: Scope.h:41
field_iterator field_begin() const
Definition: Decl.cpp:4145
This represents &#39;final&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:330
bool isOpenMPTargetExecutionDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a target code offload directive.
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents &#39;#pragma omp teams&#39; directive.
Definition: StmtOpenMP.h:2661
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
Definition: CGExpr.cpp:182
SmallVector< const Expr *, 4 > FirstprivateCopies
Expr * getInit() const
Definition: StmtOpenMP.h:805
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
This represents clause &#39;reduction&#39; in the &#39;#pragma omp ...&#39; directives.
const VarDecl * getBaseDecl(unsigned N) const
Returns the base declaration of the reduction item.
This represents &#39;#pragma omp teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:3518
void ForceCleanup(std::initializer_list< llvm::Value **> ValuesToReload={})
Force the emission of cleanups now, instead of waiting until this object is destroyed.
bool isSimple() const
Definition: CGValue.h:252
bool isOpenMPWorksharingDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a worksharing directive.
void EmitOMPSimdFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
An ordinary object is located at an address in memory.
Definition: Specifiers.h:126
SmallVector< const Expr *, 4 > ReductionOps
Controls insertion of cancellation exit blocks in worksharing constructs.
void incrementProfileCounter(const Stmt *S, llvm::Value *StepV=nullptr)
Increment the profiler&#39;s counter for the given statement by StepV.
llvm::function_ref< std::pair< llvm::Value *, llvm::Value * > CodeGenFunction &, const OMPExecutableDirective &S, Address LB, Address UB)> CodeGenDispatchBoundsTy
Represents the body of a CapturedStmt, and serves as its DeclContext.
Definition: Decl.h:4044
SmallVector< const Expr *, 4 > ReductionVars
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
Definition: Decl.h:1875
static llvm::Function * emitOutlinedFunctionPrologue(CodeGenFunction &CGF, FunctionArgList &Args, llvm::MapVector< const Decl *, std::pair< const VarDecl *, Address >> &LocalAddrs, llvm::DenseMap< const Decl *, std::pair< const Expr *, llvm::Value *>> &VLASizes, llvm::Value *&CXXThisValue, const FunctionOptions &FO)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
Definition: Decl.cpp:4404
This represents &#39;#pragma omp target parallel for simd&#39; directive.
Definition: StmtOpenMP.h:3316
ArrayRef< Expr * > private_counters()
Definition: StmtOpenMP.h:977
CompoundStmt - This represents a group of statements like { stmt stmt }.
Definition: Stmt.h:1241
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
Definition: CGDecl.cpp:1298
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1361
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents &#39;#pragma omp taskgroup&#39; directive.
Definition: StmtOpenMP.h:1971
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
Definition: CGValue.h:39
This represents clause &#39;aligned&#39; in the &#39;#pragma omp ...&#39; directives.
static ImplicitCastExpr * Create(const ASTContext &Context, QualType T, CastKind Kind, Expr *Operand, const CXXCastPath *BasePath, ExprValueKind Cat)
Definition: Expr.cpp:1844
Class intended to support codegen of all kind of the reduction clauses.
bool isGlobalVarCaptured(const VarDecl *VD) const
Checks if the global variable is captured in current function.
Expr * getCombinedLowerBoundVariable() const
Definition: StmtOpenMP.h:905
The class used to assign some variables some temporarily addresses.
This represents clause &#39;task_reduction&#39; in the &#39;#pragma omp taskgroup&#39; directives.
void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit)
Helper for the OpenMP loop directives.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
Definition: CharUnits.h:179
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
static ImplicitParamDecl * createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data, QualType Ty, CapturedDecl *CD, SourceLocation Loc)
This represents &#39;#pragma omp distribute&#39; directive.
Definition: StmtOpenMP.h:2972
This represents implicit clause &#39;depend&#39; for the &#39;#pragma omp task&#39; directive.
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
const Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
Definition: StmtOpenMP.h:196
bool addPrivate(const VarDecl *LocalVD, const llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1615
This represents &#39;proc_bind&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:675
Represent the declaration of a variable (in which case it is an lvalue) a function (in which case it ...
Definition: Decl.h:637
This represents one expression.
Definition: Expr.h:106
static void emitTargetParallelForRegion(CodeGenFunction &CGF, const OMPTargetParallelForDirective &S, PrePostActionTy &Action)
Emit only debug info necessary for generating line number tables (-gline-tables-only).
void EmitAutoVarInit(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1578
static Address invalid()
Definition: Address.h:35
bool hasLocalStorage() const
Returns true if a variable with function scope is a non-static local variable.
Definition: Decl.h:1036
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
This represents &#39;simdlen&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:501
Stmt * IgnoreContainers(bool IgnoreCaptured=false)
Skip no-op (attributed, compound) container stmts and skip captured stmt at the top, if IgnoreCaptured is true.
Definition: Stmt.cpp:147
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
llvm::function_ref< void(CodeGenFunction &, SourceLocation, const unsigned, const bool)> CodeGenOrderedTy
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
Definition: CGValue.h:66
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
const T * castAs() const
Member-template castAs<specific type>.
Definition: Type.h:6811
static void emitTargetTeamsDistributeParallelForRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S, PrePostActionTy &Action)
bool hasCancel() const
Return true if current directive has inner cancel directive.
Definition: StmtOpenMP.h:1165
static std::pair< llvm::Value *, llvm::Value * > emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
When dealing with dispatch schedules (e.g.
This represents &#39;#pragma omp target teams distribute parallel for simd&#39; combined directive.
Definition: StmtOpenMP.h:3948
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
Definition: OpenMPKinds.h:33
llvm::PointerType * getType() const
Return the type of the pointer value.
Definition: Address.h:44
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause *> Clauses)
Definition: StmtOpenMP.h:130
This represents &#39;#pragma omp target teams distribute simd&#39; combined directive.
Definition: StmtOpenMP.h:4021
void EmitOMPSimdInit(const OMPLoopDirective &D, bool IsMonotonic=false)
Helpers for the OpenMP loop directives.
This represents &#39;ordered&#39; clause in the &#39;#pragma omp ...&#39; directive.
child_range children()
Definition: Stmt.cpp:1260
Expr * getDistInc() const
Definition: StmtOpenMP.h:893
Expr * getNextLowerBound() const
Definition: StmtOpenMP.h:857
QualType getType() const
Definition: Expr.h:128
Expr * getPrevEnsureUpperBound() const
Definition: StmtOpenMP.h:899
This represents &#39;#pragma omp for&#39; directive.
Definition: StmtOpenMP.h:1104
static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDirective &S)
SmallVector< const Expr *, 4 > FirstprivateVars
static CapturedDecl * Create(ASTContext &C, DeclContext *DC, unsigned NumParams)
Definition: Decl.cpp:4454
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static std::pair< llvm::Value *, llvm::Value * > emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S, Address LB, Address UB)
if the &#39;for&#39; loop has a dispatch schedule (e.g.
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
Definition: CharUnits.h:197
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
Definition: CGLoopInfo.h:144
This represents &#39;#pragma omp target teams&#39; directive.
Definition: StmtOpenMP.h:3737
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
SourceLocation getEnd() const
UnaryOperator - This represents the unary-expression&#39;s (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
Definition: Expr.h:1896
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
Definition: ASTContext.h:1380
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
bool EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
This represents &#39;#pragma omp cancel&#39; directive.
Definition: StmtOpenMP.h:2776
This represents clause &#39;firstprivate&#39; in the &#39;#pragma omp ...&#39; directives.
ValueDecl * getDecl()
Definition: Expr.h:1114
const LangOptions & getLangOpts() const
ASTContext & getContext() const
static void emitCommonOMPTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, const RegionCodeGenTy &CodeGen)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
Definition: CGAtomic.cpp:1471
static std::pair< LValue, LValue > emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S)
The following two functions generate expressions for the loop lower and upper bounds in case of stati...
VarDecl * getCanonicalDecl() override
Retrieves the "canonical" declaration of the given declaration.
Definition: Decl.cpp:2026
GlobalDecl - represents a global declaration.
Definition: GlobalDecl.h:35
void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D)
Emit simple code for OpenMP directives in Simd-only mode.
This represents &#39;#pragma omp flush&#39; directive.
Definition: StmtOpenMP.h:2044
bool hasClausesOfKind() const
Returns true if the current directive has one or more clauses of a specific kind. ...
Definition: StmtOpenMP.h:162
This represents &#39;#pragma omp parallel for simd&#39; directive.
Definition: StmtOpenMP.h:1632
This represents &#39;seq_cst&#39; clause in the &#39;#pragma omp atomic&#39; directive.
This represents &#39;untied&#39; clause in the &#39;#pragma omp ...&#39; directive.
virtual bool hasBody() const
Returns true if this Decl represents a declaration for a body of code, such as a function or method d...
Definition: DeclBase.h:985
static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
The l-value was considered opaque, so the alignment was determined from a type.
llvm::function_ref< void(CodeGenFunction &, const OMPLoopDirective &, JumpDest)> CodeGenLoopTy
Expr * getLastIteration() const
Definition: StmtOpenMP.h:789
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
Definition: StmtOpenMP.h:148
bool isPostfixUpdate() const
Return true if &#39;v&#39; expression must be updated to original value of &#39;x&#39;, false if &#39;v&#39; must be updated ...
Definition: StmtOpenMP.h:2263
This represents &#39;#pragma omp target enter data&#39; directive.
Definition: StmtOpenMP.h:2404
void EmitOMPFlushDirective(const OMPFlushDirective &S)
Expr * getStrideVariable() const
Definition: StmtOpenMP.h:841
This represents &#39;num_teams&#39; clause in the &#39;#pragma omp ...&#39; directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
Definition: Expr.h:945
Expr * getCombinedDistCond() const
Definition: StmtOpenMP.h:947
const Stmt * getPreInits() const
Definition: StmtOpenMP.h:813
Kind
This captures a statement into a function.
Definition: Stmt.h:3105
static void emitTargetParallelRegion(CodeGenFunction &CGF, const OMPTargetParallelDirective &S, PrePostActionTy &Action)
static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S, PrePostActionTy &Action)
This represents &#39;#pragma omp single&#39; directive.
Definition: StmtOpenMP.h:1376
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
This represents &#39;hint&#39; clause in the &#39;#pragma omp ...&#39; directive.
void EmitOMPForDirective(const OMPForDirective &S)
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
Pseudo declaration for capturing expressions.
Definition: DeclOpenMP.h:217
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Definition: Type.h:2095
static void emitInnerParallelForWhenCombined(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
static void emitTargetParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetParallelForSimdDirective &S, PrePostActionTy &Action)
This is a basic class for representing single OpenMP executable directive.
Definition: StmtOpenMP.h:33
This represents &#39;schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
Definition: OpenMPClause.h:948
Expr * getExpr()
Get &#39;expr&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2270
OpenMPDirectiveKind
OpenMP directives.
Definition: OpenMPKinds.h:23
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
Definition: StmtOpenMP.h:1536
void EmitStmt(const Stmt *S, ArrayRef< const Attr *> Attrs=None)
EmitStmt - Emit the code for the statement.
Definition: CGStmt.cpp:46
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
Definition: Decl.h:1078
C-style initialization with assignment.
Definition: Decl.h:818
This represents &#39;#pragma omp taskwait&#39; directive.
Definition: StmtOpenMP.h:1927
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
This file defines OpenMP nodes for declarative directives.
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S)
Emit device code for the target simd directive.
This is a basic class for representing single OpenMP clause.
Definition: OpenMPClause.h:51
void EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S)
CanQualType VoidTy
Definition: ASTContext.h:1016
bool isOpenMPLoopBoundSharingDirective(OpenMPDirectiveKind Kind)
Checks if the specified directive kind is one of the composite or combined directives that need loop ...
void emitAggregateType(CodeGenFunction &CGF, unsigned N)
Emits the code for the variable-modified type, if required.
static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForDirective &S)
bool isAnyPointerType() const
Definition: Type.h:6300
virtual void Enter(CodeGenFunction &CGF)
void EmitOMPSingleDirective(const OMPSingleDirective &S)
An aligned address.
Definition: Address.h:25
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
Definition: Stmt.h:3206
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
Definition: Expr.h:249
This represents &#39;#pragma omp target&#39; directive.
Definition: StmtOpenMP.h:2288
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Expr * getV()
Get &#39;v&#39; part of the associated expression/statement.
Definition: StmtOpenMP.h:2265
Complete object dtor.
Definition: ABI.h:36
constexpr XRayInstrMask None
Definition: XRayInstr.h:38
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
This represents &#39;#pragma omp ordered&#39; directive.
Definition: StmtOpenMP.h:2099
QualType getType() const
Definition: CGValue.h:264
This represents &#39;#pragma omp target update&#39; directive.
Definition: StmtOpenMP.h:3040
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
Definition: Expr.cpp:215
static void EmitOMPTargetTeamsDistributeDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeDirective &S)
Emit device code for the target teams distribute directive.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
Definition: CGCall.h:356
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
Definition: CGValue.h:59
void setAction(PrePostActionTy &Action) const
CGFunctionInfo - Class to encapsulate the information about a function definition.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:67
LValue EmitOMPSharedLValue(const Expr *E)
Emits the lvalue for the expression with possibly captured variable.
void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S)
static ParmVarDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, StorageClass S, Expr *DefArg)
Definition: Decl.cpp:2535
Dataflow Directional Tag Classes.
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
Expr * getPrevUpperBoundVariable() const
Definition: StmtOpenMP.h:887
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
This represents &#39;device&#39; clause in the &#39;#pragma omp ...&#39; directive.
static void emitEmptyBoundParameters(CodeGenFunction &, const OMPExecutableDirective &, llvm::SmallVectorImpl< llvm::Value *> &)
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
Definition: CGValue.h:93
EvalResult is a struct with detailed info about an evaluated expression.
Definition: Expr.h:571
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
Definition: CGValue.h:499
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
SourceRange getSourceRange(const SourceRange &Range)
Returns the SourceRange of a SourceRange.
Definition: FixIt.h:34
Expr * getCond() const
Definition: StmtOpenMP.h:801
This represents &#39;#pragma omp section&#39; directive.
Definition: StmtOpenMP.h:1314
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
Definition: LangOptions.h:243
OpenMPScheduleClauseModifier M1
Definition: OpenMPKinds.h:143
This represents &#39;#pragma omp teams distribute&#39; directive.
Definition: StmtOpenMP.h:3450
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF, const OMPLoopDirective &S, CodeGenFunction::JumpDest LoopExit)
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
LValue getSharedLValue(unsigned N) const
Returns LValue for the reduction item.
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const Expr * getInit() const
Definition: Decl.h:1220
This represents &#39;#pragma omp simd&#39; directive.
Definition: StmtOpenMP.h:1039
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
Definition: CGBuilder.h:70
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
The name of a declaration.
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
const Expr * getReductionRef() const
Returns reference to the task_reduction return variable.
Definition: StmtOpenMP.h:2022
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
void EmitOMPTeamsDistributeParallelForDirective(const OMPTeamsDistributeParallelForDirective &S)
SourceLocation getEndLoc() const LLVM_READONLY
Definition: Stmt.h:3301
This represents clause &#39;linear&#39; in the &#39;#pragma omp ...&#39; directives.
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Expr * getUpdateExpr()
Get helper expression of the form &#39;OpaqueValueExpr(x) binop OpaqueValueExpr(expr)&#39; or &#39;OpaqueValueExp...
Definition: StmtOpenMP.h:2251
SourceLocation getBeginLoc() const LLVM_READONLY
Definition: Stmt.h:3297
llvm::Module & getModule() const
This represents &#39;#pragma omp atomic&#39; directive.
Definition: StmtOpenMP.h:2154
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
Expr * getCombinedInit() const
Definition: StmtOpenMP.h:923
unsigned getCollapsedNumber() const
Get number of collapsed loops.
Definition: StmtOpenMP.h:783
Expr * getCombinedNextLowerBound() const
Definition: StmtOpenMP.h:935
ArrayRef< Expr * > counters()
Definition: StmtOpenMP.h:971
static void emitTargetTeamsDistributeSimdRegion(CodeGenFunction &CGF, PrePostActionTy &Action, const OMPTargetTeamsDistributeSimdDirective &S)
void emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, llvm::function_ref< bool(CodeGenFunction &)> DefaultInit)
Performs initialization of the private copy for the reduction item.
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind)
Emit final update of reduction values to original variables at the end of the directive.
SwitchStmt - This represents a &#39;switch&#39; stmt.
Definition: Stmt.h:1886
Expr * getCombinedNextUpperBound() const
Definition: StmtOpenMP.h:941
void EmitAutoVarCleanups(const AutoVarEmission &emission)
Definition: CGDecl.cpp:1877
API for captured statement code generation.
void getOpenMPCaptureRegions(llvm::SmallVectorImpl< OpenMPDirectiveKind > &CaptureRegions, OpenMPDirectiveKind DKind)
Return the captured regions of an OpenMP directive.
Complex values, per C99 6.2.5p11.
Definition: Type.h:2477
bool isOpenMPDistributeDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a distribute directive.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
This file defines OpenMP AST classes for executable directives and clauses.
ArrayRef< Expr * > inits()
Definition: StmtOpenMP.h:983
static void EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDirective &S)
Emit device code for the target teams directive.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Definition: Address.h:52
bool isStaticLocal() const
Returns true if a variable with function scope is a static local variable.
Definition: Decl.h:1060
OpenMPScheduleClauseKind
OpenMP attributes for &#39;schedule&#39; clause.
Definition: OpenMPKinds.h:59
Expr * getNumIterations() const
Definition: StmtOpenMP.h:873
Base for LValueReferenceType and RValueReferenceType.
Definition: Type.h:2673
void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
StringRef getMangledName(GlobalDecl GD)
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
Definition: Linkage.h:32
bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD, Address TempAddr)
Sets the address of the variable LocalVD to be TempAddr in function CGF.
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
Definition: CGStmt.cpp:443
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
Definition: StmtOpenMP.h:193
Expr * getPrevLowerBoundVariable() const
Definition: StmtOpenMP.h:881
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
Definition: ASTContext.h:2070
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
Definition: CGStmt.cpp:36
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
Definition: ASTContext.h:2253
bool isLValueReferenceType() const
Definition: Type.h:6312
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Definition: SemaDecl.cpp:13954
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Definition: Stmt.cpp:1265
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
TranslationUnitDecl * getTranslationUnitDecl() const
Definition: ASTContext.h:1009
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
This represents &#39;#pragma omp target parallel&#39; directive.
Definition: StmtOpenMP.h:2521
void EmitOMPTargetTeamsDistributeDirective(const OMPTargetTeamsDistributeDirective &S)
static QualType getCanonicalParamType(ASTContext &C, QualType T)
This represents &#39;nowait&#39; clause in the &#39;#pragma omp ...&#39; directive.
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
Definition: Type.cpp:1874
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
Definition: CGStmt.cpp:463
This represents &#39;num_tasks&#39; clause in the &#39;#pragma omp ...&#39; directive.
Privates[]
Gets the list of initial values for linear variables.
Definition: OpenMPClause.h:151
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
Definition: Type.h:6099
Expr * getCombinedCond() const
Definition: StmtOpenMP.h:929
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> CondGen)
Emit final code for linear clauses.
void EmitOMPUseDevicePtrClause(const OMPClause &C, OMPPrivateScope &PrivateScope, const llvm::DenseMap< const ValueDecl *, Address > &CaptureDeviceAddrMap)
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Definition: CGExpr.cpp:1236
static void emitTargetTeamsDistributeParallelForSimdRegion(CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForSimdDirective &S, PrePostActionTy &Action)
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
SourceRange getSourceRange() const LLVM_READONLY
SourceLocation tokens are not useful in isolation - they are low level value objects created/interpre...
Definition: Stmt.cpp:276
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
Definition: CGExpr.cpp:1760
This represents &#39;#pragma omp taskloop simd&#39; directive.
Definition: StmtOpenMP.h:2906
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
Expr * getPreCond() const
Definition: StmtOpenMP.h:797
This represents &#39;dist_schedule&#39; clause in the &#39;#pragma omp ...&#39; directive.
CGCapturedStmtInfo * CapturedStmtInfo
bool isGlobalReg() const
Definition: CGValue.h:256
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
const VariableArrayType * getAsVariableArrayType(QualType T) const
Definition: ASTContext.h:2416
const llvm::function_ref< void(CodeGenFunction &, llvm::Value *, const OMPTaskDataTy &)> TaskGenTy
This represents &#39;#pragma omp sections&#39; directive.
Definition: StmtOpenMP.h:1246
This represents &#39;#pragma omp target data&#39; directive.
Definition: StmtOpenMP.h:2346
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
Definition: StmtOpenMP.h:2825
capture_range captures()
Definition: Stmt.h:3240
A reference to a declared variable, function, enum, etc.
Definition: Expr.h:1041
static RValue get(llvm::Value *V)
Definition: CGValue.h:86
bool isPointerType() const
Definition: Type.h:6296
static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetDirective &S)
Emit device code for the target directive.
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
static void EmitOMPTargetParallelForDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetParallelForDirective &S)
Emit device code for the target parallel for directive.
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
void EmitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
QualType getType() const
Definition: Decl.h:648
An l-value expression is a reference to an object with independent storage.
Definition: Specifiers.h:114
A trivial tuple used to represent a source range.
LValue - This represents an lvalue references.
Definition: CGValue.h:167
This represents &#39;#pragma omp taskyield&#39; directive.
Definition: StmtOpenMP.h:1839
This represents &#39;#pragma omp distribute parallel for simd&#39; composite directive.
Definition: StmtOpenMP.h:3179
CanQualType BoolTy
Definition: ASTContext.h:1017
Represents a C array with a specified size that is not an integer-constant-expression.
Definition: Type.h:2971
This represents &#39;#pragma omp parallel sections&#39; directive.
Definition: StmtOpenMP.h:1700
bool apply(CodeGenFunction &CGF)
Applies new addresses to the list of the variables.
SourceLocation getBegin() const
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
Definition: Stmt.h:3289
llvm::Value * getPointer() const
Definition: CGValue.h:323
void EmitOMPTaskDirective(const OMPTaskDirective &S)
SourceLocation getBodyRBrace() const
getBodyRBrace - Gets the right brace of the body, if a body exists.
Definition: DeclBase.cpp:906
bool isScalar() const
Definition: CGValue.h:52
This represents &#39;#pragma omp target parallel for&#39; directive.
Definition: StmtOpenMP.h:2581
SmallVector< const Expr *, 4 > FirstprivateInits
SourceLocation getLocation() const
Definition: DeclBase.h:418
This represents clause &#39;use_device_ptr&#39; in the &#39;#pragma omp ...&#39; directives.
static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeSimdDirective &S)
Emit device code for the target teams distribute simd directive.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
void EmitOMPTargetTeamsDistributeParallelForDirective(const OMPTargetTeamsDistributeParallelForDirective &S)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool Privatize()
Privatizes local variables previously registered as private.
static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(CodeGenModule &CGM, StringRef ParentName, const OMPTargetTeamsDistributeParallelForSimdDirective &S)
Emit device code for the target teams distribute parallel for simd directive.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
Definition: CGCall.cpp:1550
This represents &#39;#pragma omp taskloop&#39; directive.
Definition: StmtOpenMP.h:2841